from mtdata.index import Index, Entry, DatasetId

wiki_titles = """04j515tc7j2jgpj,arbg
fzrq1bg6rzx56iq,arcs
faka5afgeucyft7,arda
nc8gth1fv4zhsof,arde
73tosz4ire4um0n,arel
s08r9yw1hxcbecj,aren
jc6yai66mn6bh5l,ares
mdigquqrasof07s,arfa
7uecxuwkr5iwvgc,arfi
ib89l8dckxbq1xs,arfr
jaxmqqcmp75b2sy,arhu
drsjnpopadedd1p,arit
jayauqsn0ww8gr2,arja
a49vjagxu7bilwk,arko
gaxeih66haljo27,arnl
1k7ihxd8fq0fcda,arpl
aj4ofxjatqf11ho,arpt
81yx0jeximdh330,arro
qv7bij8dwgkr1yd,arru
aixl0t4w3nj1on8,arsv
bg09c7d4rqghae9,artr
02mpbq5rwz65x3y,arzh
uxfpkgzqc5ah3lm,bgcs
7p3aeedjatb39r8,bgda
582v4xlc878o88c,bgde
4q18h8l6gaz8ktx,bgel
cxovv7t0zhvvkw2,bgen
15pdxb256xz840o,bges
a953mwj173y878u,bgfa
pqatqz66cotz799,bgfi
tmcq40bgnmce3t2,bgfr
yvkdg9mu6ckaxc0,bghu
kck6ixrx2v9wtim,bgit
s5v37odvb4dufx6,bgja
ebgujhdjm5vkxpw,bgko
gjzywe3pmso0dkz,bgnl
lsrvwqvd2v7hnhw,bgpl
vfc3aizbjtnbtio,bgpt
gwvtxncgj2fubbt,bgro
pxdl3hkv4iyluyt,bgru
3spvq90nnjo56mi,bgsv
cc4gr923fz2p1rt,bgtr
yml3m6ibxiqy2yf,bgzh
16zdu0hrgs6e8e7,csda
aorw1fqdirusgjy,csde
5fmsqmbnxzubvry,csel
rq29gx3f6jvm0uh,csen
3hfpgi6do85m9w9,cses
pvqlxrtw5uka3lo,csfa
iny694ip5gpjw7f,csfi
ft21z297hchzxk4,csfr
gb963u2wi29o3l1,cshu
wqldo99ral75npg,csit
63lodxfy8em5j5i,csja
zy7vg2a3eitzkzc,csko
pwj6rfs7mxbzf7e,csnl
vztt19ld7l3tc1t,cspl
kn1h8o52ucfu5fz,cspt
feosoxiqv3jx4it,csro
l1p3mnctkyowh7k,csru
sbzxrai9g2u9rk7,cssv
uexjhfm7r67f4uc,cstr
bmnwumqfu08phhe,cszh
cothkbkina12fta,dade
vr20cg5995lce0f,dael
6vdl57ii4lwfl7h,daen
h4cfk24r7iukjwa,daes
dec2i92flwo0j98,dafa
palsd7wxsnj7ntb,dafi
n6lhykj6ju21u6t,dafr
d7cljp7x4yhnfkz,dahu
fpkv82d9evkg6am,dait
n3hd4roiob4ic0c,daja
lkoj2p06gw93nsi,dako
nby1fqfz28lnp81,danl
5n3al8y17k6fl1g,dapl
t83jb8kupejcq6u,dapt
httb0wod5r558dt,daro
pglodclo3p12r1q,daru
w4cga8jt3zsbli1,dasv
ngdluw3il43swf4,datr
gjzhc96anwtpdmb,dazh
hgmeyvavoges6hc,deel
h1wqni2a8uapmmt,deen
b5bk2yquuunm702,dees
ye07dzb4vemzrbw,defa
0gpeyfabsx3qkqo,defi
0pw75p9bv1850fu,defr
yrydz53ol6fmdnj,dehu
yo20zsmqiap8wx7,deit
ccnys6sziztd5gz,deja
enpulumwk4pkogg,deko
6jj8z6aq4qizumb,denl
ju6pt60z00r1q18,depl
uypl3ez2o57c3a0,dept
xk6rw8nwohcbyl8,dero
h6ed7vnn3qny5iu,deru
vvpbwtcq9pns50e,desv
l9ave94viu8nfqx,detr
hcjto7we7vacufu,dezh
jbovwht1zjfmbnm,elen
odmauxqvlzy2cqv,eles
cewd7x04kkxke15,elfa
uzm3aa25e6eg53z,elfi
e4qg1a161cl587n,elfr
p0xajma22nr9idl,elhu
14e4czry96r47kt,elit
c03d5wwn1t7vnar,elja
24zzf9erxrwstlo,elko
y54cqp4oiq44jvq,elnl
qec37lk6cmr6n7h,elpl
ew1rhmr1r8bqxw8,elpt
xwitiqqw5wx8cpe,elro
lrgzd4ndp1ivojo,elru
dg2l31x6k7t3n7j,elsv
5yslv5wrc6076ri,eltr
03ox5ay1l2eq1nb,elzh
7kb21jpaegtdprs,enes
km8zipw2k7bb6bz,enfa
vswatv073ncvmbv,enfi
ic8gp6kwi64jkgw,enfr
5n7o2rqrmjr9lxe,enhu
htkvdpm83803w8g,enit
6qpcq4jgm4mjfo5,enja
vz7k6t3z06vm96m,enko
p1cvbteuzs4hhq9,ennl
fgt38mk22kgjzv5,enpl
7hxuj0kmd51ltgs,enpt
rm1zju1m7ycw176,enro
13zhw7fnoioh00q,enru
h3tepdu06qcg17j,ensv
n1fjhk9lpfql9ns,entr
9d0polb8rude9ha,enzh
hrhcc6avil1e8ck,esfa
c7dxm02j955deey,esfi
t2j3216m8it8eel,esfr
6jcx1vnsuk2d8fk,eshu
eudnt94d1borcbz,esit
boem8qd3hw75mhm,esja
st9jp8b4s3kpwm4,esko
h7x8ibovv5ntf2p,esnl
7bgtl8w1ycquvxj,espl
z6b0r3ak8knjvf1,espt
oeblr9pal4svdtu,esro
4ylw6i1omeft7z5,esru
e0h78kv29s66k14,essv
eafh77vy898ip3p,estr
n3aoersqta9a333,eszh
etr6f5717dmq1pj,fafi
oqh9o2rugeb089e,fafr
xfwh15uuzp0cegk,fahu
5n63tf7wylm3cxa,fait
kx9d806liuotmfq,faja
a3w6u0x0enpzyj0,fako
jkbjfk74l8caqg5,fanl
wpp3e0hd8tdsoyy,fapl
3ywvk747hxvra5r,fapt
ugtsfl4qgsv4113,faro
usd1hp2uu218k7l,faru
mp3rvp4iezgo6pj,fasv
taihf6u5qb6b83z,fatr
t3podec9xy4dwix,fazh
x8s0qaqibz6lltn,fifr
elqgtz8pesw1h8h,fihu
brcaedpruz1tces,fiit
s1e4e3y6hnqk2g4,fija
nkyo12831b2yslp,fiko
0r3lubi5d44qqr1,finl
z2d3wy7mvioepuf,fipl
7mnut4bhh1hsejy,fipt
0u2wko45er5c90z,firo
u7df3hd2hpp1yns,firu
73mgwa15eh2rjp3,fisv
ozif8nv89vqer81,fitr
0w9btvcj6kdnteh,fizh
9lqe6u1hz1zevr2,frhu
03sbabvthbqgdzz,frit
rv5uz7hkveqeql8,frja
olcp01loge9kmo9,frko
2pd43xmpacn4k2r,frnl
ckbwgjiu8191jyl,frpl
8on0fd3wd38jw4j,frpt
1imns16wcgv7595,frro
t47k1qm2yeyysiu,frru
3xtjph0tvay2ecz,frsv
inkoca7pf9bcpt6,frtr
hxyx23pncd7j2y2,frzh
yn30cv2swhomlue,huit
yix222t5lrx47yy,huja
cm8z4rn8kii78ji,huko
5mlxkl7rue9wtfs,hunl
5va5qcnlpptxquk,hupl
2lt9ffvp0bordpt,hupt
mlnxm1h83dxuxn0,huro
rl7qh7ll73nz1qo,huru
att8du4i79kvnj4,husv
6hvq1dr7pwm33dw,hutr
j6h63j2xgwndy1r,huzh
jkyf7ewjrlr8798,itja
vwmra6xuezw7w8f,itko
zl68e5u9orkyvch,itnl
mtzh6t0igk4ge5n,itpl
5n8wob9spf5db07,itpt
0zg9xed1qnsbu6u,itro
a9n293jqwi11gpr,itru
4s3e0tmo8cu7cen,itsv
l7nt03iokcxu11b,ittr
ycmny5cy1p02jkx,itzh
03lgeo6yugf5jht,jako
7foo73ru8ilmlwb,janl
u1rlz7gsih79pg0,japl
s50ygbigtxxbbqt,japt
iyl9mlknrunzcta,jaro
h16okcs9x8a4l5w,jaru
9agu7u5jjazx8d9,jasv
ukg1ek63mjx0g04,jatr
aqchhx3krswmqzn,jazh
j8fzv0ckxpxixvf,konl
wqg4yvbxe8dhw2x,kopl
teeefh0ss6h00cs,kopt
p5rkkhe320xh3f0,koro
k1kl5q3nzqhqiny,koru
ri1tsfzdkcytjx1,kosv
bg3ccbck5skhob0,kotr
gpve56kb43lnvy6,kozh
z4to6kujh8rwss5,nlpl
lqri0zguqy2424c,nlpt
zty96o9d9390usv,nlro
ck75o5t2zu3e6nr,nlru
meyy1k475gc8ykn,nlsv
riusgvccxbsiy0l,nltr
31epwbom2i6e3p0,nlzh
612ii879zckadtf,plpt
7qdtp2c5qnop0a8,plro
yq5cn5txawdvum5,plru
iqmq7ej3nyd71kj,plsv
e73hv4ejwktqtus,pltr
200xb0bkk1ti0x6,plzh
0wbt2u9rm4l4j5y,ptro
ab5dfdl07j06dw3,ptru
hkc20tgy5ifldof,ptsv
ljbxam0f2l7kkgu,pttr
0eegtffi5xemfnl,ptzh
63pt63fwgh219k8,roru
orhildoo8ozxap7,rosv
jc4c1uzh6ftq9o9,rotr
sow4dk9p00xxslg,rozh
huiyn8hjpq5lizv,rusv
8lx5tz7x1cyil3d,rutr
jcbphinsu1oedtb,ruzh
ldlxxbhembwmoec,svtr
f6lhyojcnua3od6,svzh
a4iffiidjb8he13,trzh"""


def load_all(index: Index):
    url_ptn = 'https://www.dropbox.com/s/{uid}/wikititles-2014_{l1}{l2}.tgz?dl=1'
    rows = [row.split(',') for row in wiki_titles.splitlines()]
    for row in rows:
        uid, pair = row
        assert len(pair) == 4
        l1, l2 = pair[:2], pair[2:]
        url = url_ptn.format(uid=uid, l1=l1, l2=l2)
        in_file = f'wikititles-2014_{l1}{l2}'
        ent = Entry(did=DatasetId(group='LinguaTools', name=f'wikititles', version='2014', langs=(l1, l2)),
                    url=url, ext='tgz', in_ext='txt', in_paths=[f'{in_file}.{l1}', f'{in_file}.{l2}'])
        index.add_entry(ent)


if __name__ == '__main__':
    index = Index()
    load_all(index)
    print("loaded", len(index))
