Skip to content

Commit

Permalink
Merge pull request #6 from akb89/dev
Browse files Browse the repository at this point in the history 
Dev
  • Loading branch information
@akb89
akb89 authored Mar 22, 2017
2 parents 8efbfa6 + b9ff8a0 commit 2869ec3
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 82 deletions.
5 changes: 4 additions & 1 deletion .eslintrc.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,11 @@ rules:
#Ignore unused-var error message on chai.should()
no-unused-vars: [error, {varsIgnorePattern: 'should'}]

#Allow for of loops as forEach does not work with async await
#Allow for of loops as forEach does not work with async await (yet)
no-restricted-syntax: [warn, ForOfStatement, ForInStatement]

#Allow extending Array with chunk method for readability
no-extend-native: [warn]

#Allow param-reassign on annoSetLabelMap in importFullTexts:
no-param-reassign: [error, {props: true, ignorePropertyModificationsFor: [annoSetLabelMap]}]
10 changes: 5 additions & 5 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "noframenet",
"version": "2.0.3",
"version": "2.0.8",
"description": "A set of scripts to import FrameNet XML data to a MongoDB database",
"keywords": [
"FrameNet",
Expand Down Expand Up @@ -33,15 +33,15 @@
"bluebird": "^3.5.0",
"jsonix": "^2.4.1",
"mongodb": "^2.2.24",
"mongoose": "^4.8.6",
"mongoose": "^4.9.0",
"noframenet-core": "^3.1.0",
"winston": "^2.3.1"
},
"devDependencies": {
"babel-cli": "^6.23.0",
"babel-core": "^6.23.1",
"babel-cli": "^6.24.0",
"babel-core": "^6.24.0",
"babel-polyfill": "^6.23.0",
"babel-preset-es2015": "^6.22.0",
"babel-preset-es2015": "^6.24.0",
"babel-preset-stage-0": "^6.22.0",
"chai": "^3.5.0",
"eslint": "^3.17.1",
Expand Down
88 changes: 43 additions & 45 deletions scripts/importFullTexts.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,93 +36,88 @@ function isValidFNAnnoSet(jsonixAnnoSet) {
return false;
}

function addGFLabels(annoSetLabelMap) {
if (!annoSetLabelMap.annoSet || annoSetLabelMap.labelMap.size === 0) {
return annoSetLabelMap;
function addGFLabels(jsonixAnnoSet, labelMap) {
if (!jsonixAnnoSet || labelMap.size === 0) {
return new Map();
}
for (const jsonixLayer of toJsonixLayerArray(annoSetLabelMap.annoSet)) {
const gfLabelMap = new Map(labelMap);
for (const jsonixLayer of toJsonixLayerArray(jsonixAnnoSet)) {
if (jsonixLayer.name === 'GF') {
for (const jsonixLabel of toJsonixLabelArray(jsonixLayer)) {
if (jsonixLabel.start && jsonixLabel.end) {
if (jsonixLabel.start !== undefined && jsonixLabel.end !== undefined) {
const key = `${jsonixLabel.start}#${jsonixLabel.end}#${jsonixLayer.rank}`;
if (!annoSetLabelMap.labelMap.has(key)) {
if (!gfLabelMap.has(key)) {
// This is probably an annotation error
logger.verbose(`annotation error: GF with no FE and/or PT on #${annoSetLabelMap.annoSet.id} and layer ${jsonixLayer.name} and label ${JSON.stringify(key)}`);
return {
annoSet: annoSetLabelMap.jsonixAnnoSet,
labelMap: new Map(),
};
logger.verbose(`annotation error: GF with no FE and/or PT on #${jsonixAnnoSet.id} and layer ${jsonixLayer.name} and label ${JSON.stringify(key)}`);
return new Map();
}
annoSetLabelMap.labelMap.get(key).GF = jsonixLabel.name;
gfLabelMap.get(key).GF = jsonixLabel.name;
}
}
}
}
return annoSetLabelMap;
return gfLabelMap;
}

function addPTLabels(annoSetLabelMap) {
if (!annoSetLabelMap.annoSet || annoSetLabelMap.labelMap.size === 0) {
return annoSetLabelMap;
function addPTLabels(jsonixAnnoSet, labelMap) {
if (!jsonixAnnoSet || labelMap.size === 0) {
return new Map();
}
for (const jsonixLayer of toJsonixLayerArray(annoSetLabelMap.annoSet)) {
const ptLabelMap = new Map(labelMap);
for (const jsonixLayer of toJsonixLayerArray(jsonixAnnoSet)) {
if (jsonixLayer.name === 'PT') {
for (const jsonixLabel of toJsonixLabelArray(jsonixLayer)) {
if (jsonixLabel.start && jsonixLabel.end) {
if (jsonixLabel.start !== undefined && jsonixLabel.end !== undefined) {
const key = `${jsonixLabel.start}#${jsonixLabel.end}#${jsonixLayer.rank}`;
if (!annoSetLabelMap.labelMap.has(key)) {
if (!ptLabelMap.has(key)) {
// This is probably an annotation error
logger.verbose(`annotation error: PT with no FE on #${annoSetLabelMap.annoSet.id} and layer ${jsonixLayer.name} and label ${JSON.stringify(key)}`);
return {
annoSet: annoSetLabelMap.jsonixAnnoSet,
labelMap: new Map(),
};
logger.verbose(`annotation error: PT with no FE on #${jsonixAnnoSet.id} and layer ${jsonixLayer.name} and label ${JSON.stringify(key)}`);
return new Map();
}
annoSetLabelMap.labelMap.get(key).PT = jsonixLabel.name;
ptLabelMap.get(key).PT = jsonixLabel.name;
}
}
}
}
return annoSetLabelMap;
return ptLabelMap;
}

function addFELabels(annoSetLabelMap) {
if (!annoSetLabelMap.annoSet) {
return annoSetLabelMap;
function addFELabels(jsonixAnnoSet, labelMap) {
if (!jsonixAnnoSet) {
return new Map();
}
for (const jsonixLayer of toJsonixLayerArray(annoSetLabelMap.annoSet)) {
for (const jsonixLayer of toJsonixLayerArray(jsonixAnnoSet)) {
if (jsonixLayer.name === 'FE') {
for (const jsonixLabel of toJsonixLabelArray(jsonixLayer)) {
if (jsonixLabel.start && jsonixLabel.end) {
if (jsonixLabel.start !== undefined && jsonixLabel.end !== undefined) {
const key = `${jsonixLabel.start}#${jsonixLabel.end}#${jsonixLayer.rank}`;
if (annoSetLabelMap.labelMap.has(key)) {
if (labelMap.has(key)) {
// Do not process cases where multiple FE labels have
// same start/end values
logger.verbose(`annotation error: multiple FE labels with same start/end values on #${annoSetLabelMap.annoSet.id} and layer ${jsonixLayer.name} and label ${JSON.stringify(key)}`);
return {
annoSet: annoSetLabelMap.jsonixAnnoSet,
labelMap: new Map(),
};
logger.verbose(`annotation error: multiple FE labels with same start/end values on #${jsonixAnnoSet.id} and layer ${jsonixLayer.name} and label ${JSON.stringify(key)}`);
return new Map();
}
const value = {
FE: jsonixLabel.feID,
};
if (jsonixLabel.itype) {
if (jsonixLabel.itype !== undefined) {
value.PT = jsonixLabel.itype;
}
annoSetLabelMap.labelMap.set(key, value);
labelMap.set(key, value);
}
}
}
}
return annoSetLabelMap;
return labelMap;
}

// FE / PT / GF labels can come in any order
function getLabelMap(jsonixAnnoSet) {
return addGFLabels(addPTLabels(addFELabels({
annoSet: jsonixAnnoSet,
labelMap: new Map(),
}))).labelMap;
let labelMap = new Map();
labelMap = addFELabels(jsonixAnnoSet, labelMap);
labelMap = addPTLabels(jsonixAnnoSet, labelMap);
labelMap = addGFLabels(jsonixAnnoSet, labelMap);
return labelMap;
}

async function saveAnnoSets(jsonixSentence) {
Expand Down Expand Up @@ -155,6 +150,9 @@ async function saveAnnoSets(jsonixSentence) {
logger.debug(`isValidFNAnnoSet = ${jsonixAnnoSet.id}`);
// Look for pattern and add to annoSet
const labelMap = getLabelMap(jsonixAnnoSet);
labelMap.forEach((value, key) => {
logger.debug(`labelMap contains ${key}: ${JSON.stringify(value)} pair`);
});
const vus = [];
for (const value of labelMap.values()) {
logger.debug(`Looking for valenceUnit = ${JSON.stringify(value)}`);
Expand All @@ -169,7 +167,7 @@ async function saveAnnoSets(jsonixSentence) {
vus.add(newVu._id);
}
}
const pattern = await Pattern.findOne().where('valenceUnits').in(vus);
const pattern = await Pattern.findOne().where('valenceUnits').equals(vus);
if (pattern) {
logger.debug(`pattern found = ${pattern._id} with vus = ${vus}`);
annoSet.pattern = pattern._id;
Expand Down
31 changes: 0 additions & 31 deletions scripts/test.js
View file

This file was deleted.

0 comments on commit 2869ec3

Please sign in to comment.