111 lines
3.0 KiB
JavaScript
111 lines
3.0 KiB
JavaScript
const fs = require('fs');
|
|
const rl = require('readline')
|
|
|
|
const trie = true;
|
|
const trieLevel = 4; //size between cuts for trie ex 4 : "/exam/ple"
|
|
|
|
const dictPath = "./dictionary/";
|
|
const language = "en"
|
|
|
|
let stream = fs.createReadStream("./kaikki.org-dictionary-English.jsonl", 'utf-8');
|
|
|
|
let reader = rl.createInterface(stream);
|
|
|
|
const path = dictPath + language + "/";
|
|
|
|
fs.mkdirSync(path,{recursive:true});
|
|
var iter = 0;
|
|
reader.on('line', (line) => {
|
|
iter++;
|
|
console.log("Iteration Number: "+iter.toString());
|
|
let entry = JSON.parse(line);
|
|
let thispath = path + getPath(entry.word) + entry.pos + "/";
|
|
console.log(thispath);
|
|
if (!fs.existsSync(thispath)) {
|
|
initializeDir(thispath);
|
|
}
|
|
// console.log(entry);
|
|
|
|
let defintions = writeDefinitions(thispath, entry);
|
|
let sounds = writeSounds(thispath, entry);
|
|
let thesaurus = writeThesaurus(thispath, entry);
|
|
|
|
});
|
|
|
|
function getPath(word){
|
|
let path = "";
|
|
if (trie){
|
|
for (let i = 0; i < word.length; i+=trieLevel){
|
|
for (let n = 0; n < trieLevel; n++){
|
|
path += word[i+n] ?? "";
|
|
}
|
|
path += "/";
|
|
}
|
|
} else {
|
|
path = word + '/';
|
|
}
|
|
|
|
return path.toLowerCase();
|
|
}
|
|
|
|
function writeThesaurus(thispath, entry) {
|
|
|
|
}
|
|
function writeSounds(thispath, entry) {
|
|
var sounds = JSON.parse(fs.readFileSync(thispath+"sounds.json",'utf-8'));
|
|
|
|
if (entry.sounds == null){
|
|
console.log("No sounds key present on entry, skipping sounds def...");
|
|
return;
|
|
}
|
|
entry.sounds.forEach((ele,idx,arr) => {
|
|
if (ele.audio != null) {
|
|
sounds.audios.push(ele.ogg_url, ele.mp3_url);
|
|
} else if (ele.rhymes != null) {
|
|
sounds.rhymes.push(ele.rhymes)
|
|
} else if (ele.homophone != null) {
|
|
sounds.homophones.push(ele.homophone);
|
|
} else {
|
|
sounds.pronunciations.push(ele);
|
|
}
|
|
});
|
|
fs.writeFileSync(thispath+"sounds.json",JSON.stringify(sounds));
|
|
sounds = null;
|
|
}
|
|
function writeDefinitions(thispath, entry) {
|
|
|
|
var definitions = JSON.parse(fs.readFileSync(thispath+"definitions.json",'utf-8'));
|
|
|
|
let senses = entry.senses;
|
|
if (senses == null){
|
|
console.log("No senses key present on entry, skipping senses def");
|
|
return;
|
|
}
|
|
|
|
senses.forEach((ele,idx,arr) => {
|
|
//add glosses onto defnitions list
|
|
definitions.glosses.push(ele.glosses);
|
|
});
|
|
fs.writeFileSync(thispath+"definitions.json", JSON.stringify(definitions));
|
|
definitions = null;
|
|
}
|
|
function initializeDir(path) {
|
|
fs.mkdirSync(path, {recursive:true});
|
|
let definitions = {
|
|
glosses: []
|
|
}
|
|
let sounds = {
|
|
pronunciations: [],
|
|
audios: [],
|
|
rhymes: [],
|
|
homophones: []
|
|
}
|
|
let thesaurus = {
|
|
synonyms: [],
|
|
antynoms: []
|
|
}
|
|
fs.writeFileSync(path+"definitions.json",JSON.stringify(definitions));
|
|
fs.writeFileSync(path+"sounds.json",JSON.stringify(sounds));
|
|
fs.writeFileSync(path+"thesaurus.json",JSON.stringify(thesaurus));
|
|
|
|
} |