Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit 1edc0e2

Browse files
cross-train core library in bf-lu (#601)
* add qna cross training * Revert "remove cross-train cli related changes and only keep the api" This reverts commit ab1fb84. * call qnaCrossTrain * add qna cross traini * expose function * optimize and add more corner test cases * rename crossTrainer class * remove cli code pieces and optimize tests * move cross-train.ts to lu folder * optimize function headers and naming * typo: dialogname to dialogName * add more test cases * remove unused ' * add cross-train.ts cli file for test convenience and will remove it once review done * fix bugs * trigger ci * update qna cross train based on feedbacks * add config path parameter to specify the config file * fix posttest failure * remove dup utterances in interuption intents * remove dup 1 * optimize all de-dup logic * resolve all feedbacks * add test cases to test dedup functions * fix tslint errors * add more friendly description for config CLI parameter * fix config full path issue and remove uncessary loop validation * move cross train CLI to a new package named cross-train * fix build error in CI validation * add missing package in package.json * remove unused file * fix typo * optimize dedep logic and resolve feedbacks * adjust test cases * support qna source, id, prompts reconstruction in cross training * typo: interuption to interruption * revert CLI changes * minor fix * refactor cross train library tests Co-authored-by: Vishwac Sena Kannan <vishwack@hotmail.com>
1 parent 49f1871 commit 1edc0e2

8 files changed

Lines changed: 986 additions & 292 deletions

File tree

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/*!
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License.
4+
*/
5+
6+
const fs = require('fs-extra')
7+
const path = require('path')
8+
const file = require('../../utils/filehelper')
9+
const fileExtEnum = require('../utils/helpers').FileExtTypeEnum
10+
const exception = require('../utils/exception')
11+
const retCode = require('../utils/enums/CLI-errors');
12+
const crossTrainer = require('./crossTrainer')
13+
14+
module.exports = {
15+
/**
16+
* Cross train lu and qna files.
17+
* @param {string} input input lu and qna files folder.
18+
* @param {string} intentName interruption intent name. Default value is _Interruption.
19+
* @param {string} configPath path to config of mapping rules. If undefined, it will read config.json from input folder.
20+
* @returns {luResult: any, qnaResult: any} trainedResult of luResult and qnaResult or undefined if no results.
21+
*/
22+
train: async function (input, intentName, configPath) {
23+
let trainedResult
24+
25+
// Parse lu and qna objects
26+
const luObjects = await file.getLuObjects(undefined, input, true, fileExtEnum.LUFile)
27+
const qnaObjects = await file.getLuObjects(undefined, input, true, fileExtEnum.QnAFile)
28+
29+
let configObject
30+
if (configPath && configPath !== '') {
31+
configObject = await file.getConfigObject(configPath)
32+
} else {
33+
configObject = await file.getConfigObject(input)
34+
}
35+
36+
if (configObject.rootIds.length > 0) {
37+
let crossTrainConfig = {
38+
rootIds: configObject.rootIds,
39+
triggerRules: configObject.triggerRules,
40+
intentName: intentName,
41+
verbose: true
42+
}
43+
44+
trainedResult = crossTrainer.crossTrain(luObjects, qnaObjects, JSON.stringify(crossTrainConfig))
45+
} else {
46+
throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'rootDialog property is required in config file'))
47+
}
48+
49+
return trainedResult
50+
},
51+
52+
/**
53+
* Write lu and qna files
54+
* @param {Map<string, any>} fileIdToLuResourceMap lu or qna file id to lu resource map.
55+
* @param {string} out output folder name. If not specified, source lu and qna files will be updated.
56+
* @throws {exception} Throws on errors.
57+
*/
58+
writeFiles: async function (fileIdToLuResourceMap, out) {
59+
if (fileIdToLuResourceMap) {
60+
let newFolder
61+
if (out) {
62+
newFolder = out
63+
if (!path.isAbsolute(out)) {
64+
newFolder = path.resolve(out)
65+
}
66+
67+
if (!fs.existsSync(newFolder)) {
68+
fs.mkdirSync(newFolder)
69+
}
70+
}
71+
72+
for (const fileId of fileIdToLuResourceMap.keys()) {
73+
try {
74+
if (newFolder) {
75+
const fileName = path.basename(fileId)
76+
const newFileId = path.join(newFolder, fileName)
77+
await fs.writeFile(newFileId, fileIdToLuResourceMap.get(fileId).Content, 'utf-8')
78+
} else {
79+
await fs.writeFile(fileId, fileIdToLuResourceMap.get(fileId).Content, 'utf-8')
80+
}
81+
} catch (err) {
82+
throw (new exception(retCode.errorCode.OUTPUT_FOLDER_INVALID, `Unable to write to file ${fileId}. Error: ${err.message}`))
83+
}
84+
}
85+
}
86+
}
87+
}

0 commit comments

Comments
 (0)