Skip to content

Commit ab83c7c

Browse files
brollbbrollb
authored
Update static input support for credentials. Fixes #1862 (#1871)
* Update static input support for credentials. Fixes #1862 * remove unused code * Fix linting issues
1 parent 64e5266 commit ab83c7c

2 files changed

Lines changed: 245 additions & 192 deletions

File tree

‎src/plugins/Export/Export.js‎

Lines changed: 139 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ define([
1818
CONSTANTS,
1919
BlobConfig,
2020
_,
21-
Q
2221
) {
2322
'use strict';
2423

@@ -71,7 +70,7 @@ define([
7170
*
7271
* @param {function(string, plugin.PluginResult)} callback - the result callback
7372
*/
74-
Export.prototype.main = function (callback) {
73+
Export.prototype.main = async function (callback) {
7574
this.resetVariableNames();
7675
this.dataInputs = {};
7776
this.dataOutputs = {};
@@ -84,17 +83,17 @@ define([
8483

8584
const files = new GeneratedFiles(this.blobClient);
8685
const name = this.core.getAttribute(this.activeNode, 'name');
87-
const staticInputs = this.getCurrentConfig().staticInputs;
88-
return this.createPipelineFiles(this.activeNode, files)
89-
.then(() => this.addStaticInputs(staticInputs, files))
90-
.then(() => this.createDefaultMainFile(this.activeNode, staticInputs, files))
91-
.then(() => files.save(name))
92-
.then(hash => {
93-
this.result.addArtifact(hash);
94-
this.result.setSuccess(true);
95-
callback(null, this.result);
96-
})
97-
.catch(err => callback(err, this.result));
86+
const staticInputDict = this.getCurrentConfig().staticInputs;
87+
await this.createPipelineFiles(this.activeNode, files);
88+
89+
const staticInputData = Object.values(staticInputDict);
90+
await Promise.all(staticInputData.map(input => this.addStaticInput(input, files)));
91+
await this.createDefaultMainFile(this.activeNode, staticInputDict, files);
92+
93+
const hash = await files.save(name);
94+
this.result.addArtifact(hash);
95+
this.result.setSuccess(true);
96+
callback(null, this.result);
9897
};
9998

10099
Export.prototype.resetVariableNames = function () {
@@ -115,7 +114,13 @@ define([
115114
return name;
116115
};
117116

118-
Export.prototype.getVariableNameFor = function (nodeId) {
117+
Export.prototype.getVariableNameFor = async function (nodeId) {
118+
if (!this.variableNameFor[nodeId]) {
119+
const node = await this.core.loadByPath(this.rootNode, nodeId);
120+
const basename = this.core.getAttribute(node, 'name')
121+
.replace(/[^a-zA-Z0-9]/g, '_');
122+
this.assignVariableTo(basename, nodeId);
123+
}
119124
return this.variableNameFor[nodeId];
120125
};
121126

@@ -128,61 +133,59 @@ define([
128133
return varName;
129134
};
130135

131-
Export.prototype.addStaticInputs = function (ids, files) {
136+
Export.prototype.addStaticInput = async function (info, files) {
132137
// Get the static inputs and add them in artifacts/
133-
return Q.all(ids.map(id => this.core.loadByPath(this.rootNode, id)))
134-
.then(nodes => {
135-
nodes.forEach((node, i) => {
136-
const name = this.getVariableNameFor(ids[i]);
137-
const dataInfo = this.getAttribute(node, 'data');
138-
files.addUserAsset(`artifacts/${name}`, dataInfo);
139-
});
140-
return files;
141-
});
138+
const node = await this.core.loadByPath(this.rootNode, info.id);
139+
const name = await this.getVariableNameFor(info.id);
140+
const dataInfo = this.core.getAttribute(node, 'data');
141+
files.addUserAsset(`artifacts/${name}`, dataInfo, info.credentials);
142+
return files;
142143
};
143144

144-
Export.prototype.createDefaultMainFile = function (node, staticInputs, files) {
145+
Export.prototype.createDefaultMainFile = function (node, staticInputDict, files) {
145146
// Get the variable name for the pipeline
146147
const name = PluginBase.toUpperCamelCase(this.core.getAttribute(node, 'name'));
147148
const instanceName = this.getVariableName(name.toLowerCase());
148149
let initCode = null;
149150
return this.getAllInitialCode()
150151
.then(code => initCode = code)
151152
.then(() => this.core.loadChildren(node))
152-
.then(nodes => {
153+
.then(async nodes => {
153154
// Get code for each input
154155
const inputs = this.getPipelineInputs(nodes);
155-
const inputNames = inputs.map(input => this.getVariableNameFor(input[1]));
156+
const inputNames = await Promise.all(inputs.map(input => this.getVariableNameFor(input[1])));
156157
let argIndex = 1;
157-
const parseInputCode = inputs.map((input, i) => {
158+
const parseInputCode = (await Promise.all(inputs.map(async (input, i) => {
158159
const [, , node] = input;
159160
const inputName = inputNames[i];
160161
const pathNameVar = this.getVariableName(`${inputName}_path`);
161-
const type = this.getAttribute(node, 'type');
162+
const type = this.core.getAttribute(node, 'type');
162163
const id = this.core.getPath(node);
163-
const isStatic = staticInputs.includes(id);
164+
const artifactInfo = staticInputDict[id];
164165

166+
console.log(`checking if ${id} is static`, staticInputDict);
165167
const lines = [
166168
`${inputName} = deepforge.serialization.load('${type}', open(${pathNameVar}, 'rb'))`
167169
];
168170

169-
if (isStatic) {
170-
lines.unshift(`${pathNameVar} = 'artifacts/${inputName}'`);
171+
if (artifactInfo) {
172+
const artifactName = await this.getVariableNameFor(artifactInfo.id);
173+
lines.unshift(`${pathNameVar} = 'artifacts/${artifactName}'`);
171174
} else {
172175
lines.unshift(`${pathNameVar} = sys.argv[${argIndex}]`);
173176
argIndex++;
174177
}
175178
return lines.join('\n');
176-
}).join('\n');
179+
}))).join('\n');
177180

178181
// Create code for saving outputs to outputs/
179182
const outputs = this.getPipelineOutputs(nodes);
180-
const outputNames = outputs.map(output => this.getVariableNameFor(output[1]));
183+
const outputNames = await Promise.all(outputs.map(output => this.getVariableNameFor(output[1])));
181184

182185
const saveNames = outputs.map(output => {
183186
const [, , node] = output;
184187
const outputOp = this.core.getParent(this.core.getParent(node));
185-
return this.getAttribute(outputOp, 'saveName');
188+
return this.core.getAttribute(outputOp, 'saveName');
186189
});
187190
const printResults = outputNames
188191
.map((name, i) => `print(' ${saveNames[i]}: ' + str(${name}))`);
@@ -233,120 +236,115 @@ define([
233236
});
234237
};
235238

236-
Export.prototype.createPipelineFiles = function (node, files) {
239+
Export.prototype.createPipelineFiles = async function (node, files) {
237240
const name = PluginBase.toUpperCamelCase(this.core.getAttribute(node, 'name'));
238241
// Generate the file for the pipeline in pipelines/
239242

240243
let allOperations,
241244
operations,
242245
connections;
243246

244-
return this.core.loadChildren(node)
245-
.then(nodes => { // Assign variable names to all data
246-
const promises = nodes
247-
.filter(node => this.isMetaTypeOf(node, this.META.Operation))
248-
.map(operation => this.cacheDataNodes(operation));
249-
250-
return Q.all(promises).then(() => nodes);
251-
})
252-
.then(nodes => {
253-
254-
// Get the important node types and get all the code for the operations
255-
allOperations = this.getSortedOperations(nodes);
256-
operations = allOperations
257-
.filter(node => !this.isMetaTypeOf(node, this.META.Input))
258-
.filter(node => !this.isMetaTypeOf(node, this.META.Output));
259-
260-
// For each operation, instantiate it with the respective arguments
261-
connections = nodes
262-
.filter(node => !this.isMetaTypeOf(node, this.META.Operation));
263-
264-
connections.forEach(conn => {
265-
const srcId = this.core.getPointerPath(conn, 'src');
266-
const dstId = this.core.getPointerPath(conn, 'dst');
267-
// Get the src data name?
268-
// TODO
269-
this.assignVariableTo('result', srcId, dstId);
270-
});
247+
const nodes = await this.core.loadChildren(node);
248+
const promises = nodes
249+
.filter(node => this.isMetaTypeOf(node, this.META.Operation))
250+
.map(operation => this.cacheDataNodes(operation));
251+
252+
await Promise.all(promises);
253+
// Get the important node types and get all the code for the operations
254+
allOperations = this.getSortedOperations(nodes);
255+
operations = allOperations
256+
.filter(node => !this.isMetaTypeOf(node, this.META.Input))
257+
.filter(node => !this.isMetaTypeOf(node, this.META.Output));
258+
259+
// For each operation, instantiate it with the respective arguments
260+
connections = nodes
261+
.filter(node => !this.isMetaTypeOf(node, this.META.Operation));
262+
263+
connections.forEach(conn => {
264+
const srcId = this.core.getPointerPath(conn, 'src');
265+
const dstId = this.core.getPointerPath(conn, 'dst');
266+
// Get the src data name?
267+
// TODO
268+
this.assignVariableTo('result', srcId, dstId);
269+
});
271270

272-
return Q.all(operations.map(operation => this.createOperation(operation)));
273-
})
274-
.then(operationOutputs => {
275-
let code = [];
276-
277-
operationOutputs.forEach(output => {
278-
// Create the operation
279-
const [lines, opName, operation] = output;
280-
code = lines.concat(code);
281-
282-
// execute it
283-
284-
// Get the inputs of the operation
285-
let inputs = this.getCachedInputs(operation)
286-
.map(tuple => {
287-
const [, id] = tuple;
288-
const srcId = this.getSrcDataId(connections, id);
289-
return this.getVariableNameFor(srcId);
290-
})
291-
.join(',');
292-
293-
// Get the outputs of the operation (assign variable names)
294-
let outputs = this.getCachedOutputs(operation)
295-
.map(tuple => {
296-
const [, id] = tuple;
297-
const variable = this.getVariableNameFor(id);
298-
return variable;
299-
})
300-
.filter(name => !!name)
301-
.join(',');
302-
303-
if (outputs) {
304-
code.unshift(`${outputs} = ${opName}.execute(${inputs})`);
305-
} else {
306-
code.unshift(`${opName}.execute(${inputs})`);
307-
}
308-
});
271+
const createOps = operations.map(operation => this.createOperation(operation));
272+
const operationOutputs = await Promise.all(createOps);
273+
let code = [];
274+
275+
for (let i = 0; i < operationOutputs.length; i++) {
276+
const output = operationOutputs[i];
277+
const [lines, opName, operation] = output;
278+
code = lines.concat(code);
279+
280+
// execute it
281+
282+
// Get the inputs of the operation
283+
let inputs = (await Promise.all(this.getCachedInputs(operation)
284+
.map(tuple => {
285+
const [, id] = tuple;
286+
const srcId = this.getSrcDataId(connections, id);
287+
return this.getVariableNameFor(srcId);
288+
})))
289+
.join(',');
290+
291+
// Get the outputs of the operation (assign variable names)
292+
const outputs = await this.getOutputs(operation);
293+
const outputNames = (await Promise.all(
294+
outputs.map(async tuple => {
295+
const [, id] = tuple;
296+
const variable = await this.getVariableNameFor(id);
297+
return variable;
298+
})))
299+
.filter(name => !!name)
300+
.join(',');
301+
302+
if (outputNames) {
303+
code.unshift(`${outputNames} = ${opName}.execute(${inputs})`);
304+
} else {
305+
code.unshift(`${opName}.execute(${inputs})`);
306+
}
307+
}
309308

310-
// Import each operation
311-
let operationTypes = operations.map(node => {
312-
const base = this.core.getBase(node);
313-
return this.core.getAttribute(base, 'name');
314-
});
315-
operationTypes = _.uniq(operationTypes);
316-
operationTypes.forEach(type => code.unshift(`from operations import ${type}\n`));
317-
318-
319-
// Create the pipeline file
320-
const inputs = this.getPipelineInputs(allOperations)
321-
.map(tuple => this.getVariableNameFor(tuple[1]))
322-
.join(', ');
323-
const outputs = this.getPipelineOutputs(allOperations)
324-
.map(tuple => this.getVariableNameFor(tuple[1]))
325-
.filter(name => !!name)
326-
.join(', ');
327-
328-
// Move imports to the top
329-
const importCode = code.filter(line => line.includes('import'));
330-
code = code.filter(line => !line.includes('import'));
331-
332-
// Move all operation construction to the front
333-
const opInvocations = code.filter(line => line.includes('execute'));
334-
code = code.filter(line => !line.includes('execute'));
335-
code = code.concat(opInvocations);
336-
337-
const filename = PluginBase.toSnakeCase(name);
338-
const pipelinePy = [
339-
importCode.join('\n'),
340-
'',
341-
`class ${name}():`,
342-
indent(`def execute(self${inputs && ', '}${inputs}):`),
343-
indent(indent(code.join('\n'))),
344-
indent(indent(`return ${outputs}`))
345-
].join('\n');
346-
files.addFile(`pipelines/${filename}.py`, pipelinePy);
347-
files.appendToFile('pipelines/__init__.py', `from pipelines.${filename} import ${name}\n`);
348-
return Q.all(operations.map(node => this.createOperationFiles(node, files)));
349-
});
309+
// Import each operation
310+
let operationTypes = operations.map(node => {
311+
const base = this.core.getBase(node);
312+
return this.core.getAttribute(base, 'name');
313+
});
314+
operationTypes = _.uniq(operationTypes);
315+
operationTypes.forEach(type => code.unshift(`from operations import ${type}\n`));
316+
317+
318+
// Create the pipeline file
319+
const inputs = (await Promise.all(this.getPipelineInputs(allOperations)
320+
.map(tuple => this.getVariableNameFor(tuple[1]))))
321+
.join(', ');
322+
const outputs = (await Promise.all(this.getPipelineOutputs(allOperations)
323+
.map(tuple => this.getVariableNameFor(tuple[1]))))
324+
.filter(name => !!name)
325+
.join(', ');
326+
327+
// Move imports to the top
328+
const importCode = code.filter(line => line.includes('import'));
329+
code = code.filter(line => !line.includes('import'));
330+
331+
// Move all operation construction to the front
332+
const opInvocations = code.filter(line => line.includes('execute'));
333+
code = code.filter(line => !line.includes('execute'));
334+
code = code.concat(opInvocations);
335+
336+
const filename = PluginBase.toSnakeCase(name);
337+
const pipelinePy = [
338+
importCode.join('\n'),
339+
'',
340+
`class ${name}():`,
341+
indent(`def execute(self${inputs && ', '}${inputs}):`),
342+
indent(indent(code.join('\n'))),
343+
indent(indent(`return ${outputs}`))
344+
].join('\n');
345+
files.addFile(`pipelines/${filename}.py`, pipelinePy);
346+
files.appendToFile('pipelines/__init__.py', `from pipelines.${filename} import ${name}\n`);
347+
return Promise.all(operations.map(node => this.createOperationFiles(node, files)));
350348
};
351349

352350
Export.prototype.getPipelineInputs = function (nodes) {
@@ -495,7 +493,7 @@ define([
495493

496494
Export.prototype.getCurrentConfig = function () {
497495
var config = PluginBase.prototype.getCurrentConfig.call(this);
498-
config.staticInputs = config.staticInputs || [];
496+
config.staticInputs = config.staticInputs || {};
499497
return config;
500498
};
501499

0 commit comments

Comments
 (0)