From 47c07746ecdb103f8760bcebd838a79d4b1fd0d5 Mon Sep 17 00:00:00 2001 From: ZhangGe6 Date: Fri, 3 Jun 2022 21:40:45 +0800 Subject: [PATCH] the dropdown menu of adding new node is basically done --- docs/onnx_modifier_todo.md | 25 ++++++++++++++++ onnx_modifier.py | 9 ++---- readme.md | 4 +-- static/index.js | 2 +- static/onnx.js | 59 +++++++++++++++++++++++++++++++++----- static/view-sidebar.js | 38 +++++++++++++++++------- 6 files changed, 110 insertions(+), 27 deletions(-) create mode 100644 docs/onnx_modifier_todo.md diff --git a/docs/onnx_modifier_todo.md b/docs/onnx_modifier_todo.md new file mode 100644 index 0000000..a22afb1 --- /dev/null +++ b/docs/onnx_modifier_todo.md @@ -0,0 +1,25 @@ +# Add node + +extend shape: https://github.com/onnx/onnx/issues/3920 + +add node (extend output's dim): https://github.com/onnx/onnx/issues/2709 + +add node （NMS）: https://github.com/onnx/onnx/issues/2216 + +add node (add preprocess nodes): https://zhuanlan.zhihu.com/p/394395167 + +# modify attribute of nodes + +topk: https://github.com/onnx/onnx/issues/2921 + + + +# done + +remove layer: https://github.com/onnx/onnx/issues/2638 + + + +# 或许可以帮助 + +http://yyixx.com/docs/algo/onnx/ \ No newline at end of file diff --git a/onnx_modifier.py b/onnx_modifier.py index 715a6d3..c23a26f 100644 --- a/onnx_modifier.py +++ b/onnx_modifier.py @@ -119,13 +119,9 @@ if __name__ == "__main__": def remove_node_by_node_states(): print(len(onnx_modifier.graph.node)) print(len(onnx_modifier.graph.initializer)) - node_states_fp = {'data_0': 'Exist', 'Conv0': 'Exist', 'Relu1': 'Exist', 'MaxPool2': 'Exist', 'Conv3': 'Exist', 'Relu4': 'Exist', 'Conv5': 'Exist', 'Relu6': 'Exist', 'Conv7': 'Deleted', 'Relu8': 'Deleted', 'Concat9': 'Deleted', 'Conv10': 'Deleted', 'Relu11': 'Deleted', 'Conv12': 'Deleted', 'Relu13': 'Deleted', 'Conv14': 'Deleted', 'Relu15': 'Deleted', 'Concat16': 'Deleted', 'MaxPool17': 'Deleted', 'Conv18': 'Deleted', 'Relu19': 'Deleted', 'Conv20': 'Deleted', 'Relu21': 'Deleted', 'Conv22': 'Deleted', 'Relu23': 'Deleted', 'Concat24': 'Deleted', 'Conv25': 'Deleted', 'Relu26': 'Deleted', 'Conv27': 'Deleted', 'Relu28': 'Deleted', 'Conv29': 'Deleted', 'Relu30': 'Deleted', 'Concat31': 'Deleted', 'MaxPool32': 'Deleted', 'Conv33': 'Deleted', 'Relu34': 'Deleted', 'Conv35': 'Deleted', 'Relu36': 'Deleted', 'Conv37': 'Deleted', 'Relu38': 'Deleted', 'Concat39': 'Deleted', 'Conv40': 'Deleted', 'Relu41': 'Deleted', 'Conv42': 'Deleted', 'Relu43': 'Deleted', 'Conv44': 'Deleted', 'Relu45': 'Deleted', 'Concat46': 'Deleted', 'Conv47': 'Deleted', 'Relu48': 'Deleted', 'Conv49': 'Deleted', 'Relu50': 'Deleted', 'Conv51': 'Deleted', 'Relu52': 'Deleted', 'Concat53': 'Deleted', 'Conv54': 'Deleted', 'Relu55': 'Deleted', 'Conv56': 'Deleted', 'Relu57': 'Deleted', 'Conv58': 'Deleted', 'Relu59': 'Deleted', 'Concat60': 'Deleted', 'Dropout61': 'Deleted', 'Conv62': 'Deleted', 'Relu63': 'Deleted', 'GlobalAveragePool64': 'Deleted', 'Softmax65': 'Deleted', 'softmaxout_1': 'Deleted'} - - node_states_quant = {'data_0': 'Exist', 'data_0_QuantizeLinear': 'Exist', 'Conv_nc_rename_0_quant': 'Exist', 'MaxPool_nc_rename_2_quant': 'Exist', 'Conv_nc_rename_3_quant': 'Deleted', 'Conv_nc_rename_5_quant': 'Deleted', 'Conv_nc_rename_7_quant': 'Deleted', 'fire2/expand1x1_2_DequantizeLinear': 'Deleted', 'fire2/expand3x3_2_DequantizeLinear': 'Deleted', 'Concat_nc_rename_9': 'Deleted', 'fire2/concat_1_QuantizeLinear': 'Deleted', 'Conv_nc_rename_10_quant': 'Deleted', 'Conv_nc_rename_12_quant': 'Deleted', 'Conv_nc_rename_14_quant': 'Deleted', 'fire3/expand1x1_2_DequantizeLinear': 'Deleted', - 'fire3/expand3x3_2_DequantizeLinear': 'Deleted', 'Concat_nc_rename_16': 'Deleted', 'MaxPool_nc_rename_17': 'Deleted', 'pool3_1_QuantizeLinear': - 'Deleted', 'Conv_nc_rename_18_quant': 'Deleted', 'Conv_nc_rename_20_quant': 'Deleted', 'Conv_nc_rename_22_quant': 'Deleted', 'fire4/expand1x1_2_DequantizeLinear': 'Deleted', 'fire4/expand3x3_2_DequantizeLinear': 'Deleted', 'Concat_nc_rename_24': 'Deleted', 'fire4/concat_1_QuantizeLinear': 'Deleted', 'Conv_nc_rename_25_quant': 'Deleted', 'Conv_nc_rename_27_quant': 'Deleted', 'Conv_nc_rename_29_quant': 'Deleted', 'fire5/expand1x1_2_DequantizeLinear': 'Deleted', 'fire5/expand3x3_2_DequantizeLinear': 'Deleted', 'Concat_nc_rename_31': 'Deleted', 'MaxPool_nc_rename_32': 'Deleted', 'pool5_1_QuantizeLinear': 'Deleted', 'Conv_nc_rename_33_quant': 'Deleted', 'Conv_nc_rename_35_quant': 'Deleted', 'Conv_nc_rename_37_quant': 'Deleted', 'fire6/expand1x1_2_DequantizeLinear': 'Deleted', 'fire6/expand3x3_2_DequantizeLinear': 'Deleted', 'Concat_nc_rename_39': 'Deleted', 'fire6/concat_1_QuantizeLinear': 'Deleted', 'Conv_nc_rename_40_quant': 'Deleted', 'Conv_nc_rename_42_quant': 'Deleted', 'Conv_nc_rename_44_quant': 'Deleted', 'fire7/expand1x1_2_DequantizeLinear': 'Deleted', 'fire7/expand3x3_2_DequantizeLinear': 'Deleted', 'Concat_nc_rename_46': 'Deleted', 'fire7/concat_1_QuantizeLinear': 'Deleted', 'Conv_nc_rename_47_quant': 'Deleted', 'Conv_nc_rename_49_quant': 'Deleted', 'Conv_nc_rename_51_quant': 'Deleted', 'fire8/expand1x1_2_DequantizeLinear': 'Deleted', 'fire8/expand3x3_2_DequantizeLinear': 'Deleted', 'Concat_nc_rename_53': 'Deleted', 'fire8/concat_1_QuantizeLinear': 'Deleted', 'Conv_nc_rename_54_quant': 'Deleted', 'Conv_nc_rename_56_quant': 'Deleted', 'Conv_nc_rename_58_quant': 'Deleted', 'fire9/expand1x1_2_DequantizeLinear': 'Deleted', 'fire9/expand3x3_2_DequantizeLinear': 'Deleted', 'Concat_nc_rename_60': 'Deleted', 'fire9/concat_1_QuantizeLinear': 'Deleted', 'Conv_nc_rename_61_quant': 'Deleted', 'GlobalAveragePool_nc_rename_63_quant': 'Deleted', 'pool10_1_DequantizeLinear': 'Deleted', 'Softmax_nc_rename_64': 'Deleted', 'softmaxout_1': 'Deleted'} + node_states_fp = {} + node_states_quant = {} - node_states = node_states_quant # node_states = node_states_fp # print('\graph input') @@ -171,7 +167,6 @@ if __name__ == "__main__": # for initializer in onnx_modifier.model_proto.graph.initializer: # print(initializer.name) # print(onnx_modifier.model_proto.graph.initializer['fire9/concat_1_scale']) - pass # explore_basic() def test_modify_node_io_name(): diff --git a/readme.md b/readme.md index 1e445d2..2864d93 100644 --- a/readme.md +++ b/readme.md @@ -23,7 +23,7 @@ Hope it helps! # Get started -Clone the repo and install the require Python packages by +Clone the repo and install the required Python packages by ```bash git clone git@github.com:ZhangGe6/onnx-modifier.git @@ -63,7 +63,7 @@ Node-level-operation elements are all in the sidebar, which can be invoked by cl ## Delete node -There are two modes for deleting node: `Delete With Children` and `Delete Single Node`. `Delete Single Node` only deletes the clicked node, while `Delete With Children` also deletes all the node rooted on the clicked node, which is convenient and nature if we want to delete a long path of nodes. +There are two modes for deleting node: `Delete With Children` and `Delete Single Node`. `Delete Single Node` only deletes the clicked node, while `Delete With Children` also deletes all the node rooted on the clicked node, which is convenient and natural if we want to delete a long path of nodes. > The implementation of `Delete With Children` is based on the backtracking algorithm. diff --git a/static/index.js b/static/index.js index 668316a..f1ccd6e 100644 --- a/static/index.js +++ b/static/index.js @@ -240,7 +240,7 @@ host.BrowserHost = class { swal("Success!", "Modified model has been successfuly saved in ./modified_onnx/", "success"); } else { - swal("Error happens!", "You can find it out or create an issue on https://github.com/ZhangGe6/onnx-modifier", "error"); + swal("Error happens!", "You are kindly to create an issue on https://github.com/ZhangGe6/onnx-modifier", "error"); // alert('Error happens, you can find it out or create an issue on https://github.com/ZhangGe6/onnx-modifier') } }); diff --git a/static/onnx.js b/static/onnx.js index dc489c0..59ebd9d 100644 --- a/static/onnx.js +++ b/static/onnx.js @@ -327,11 +327,14 @@ onnx.Model = class { } this._graphs = []; if (model && model.graph) { - const graphMetadata = new onnx.GraphMetadata(metadata, imports); - const context = new onnx.ModelContext(graphMetadata, imageFormat); + // const graphMetadata = new onnx.GraphMetadata(metadata, imports); + // const context = new onnx.ModelContext(graphMetadata, imageFormat); + this.graphMetadata = new onnx.GraphMetadata(metadata, imports); + const context = new onnx.ModelContext(this.graphMetadata, imageFormat); for (const func of model.functions || []) { context.metadata.add(new onnx.Function(context, func)); } + // var tmp = this.supported_ops const graphs = [ model.graph ]; while (graphs.length > 0) { const graph = graphs.shift(); @@ -403,10 +406,24 @@ onnx.Model = class { get graphs() { return this._graphs; } + + get supported_ops() { + console.log(this.graphMetadata); + var ops = [] + for (const domain of this.graphMetadata._metadata._map.keys()) { + // console.log(domain) + for (const op of this.graphMetadata._metadata._map.get(domain).keys()) { + // console.log(op) + ops.push(op) + } + } + return ops + } + }; onnx.Graph = class { - + // context is ModelContext here constructor(context, graph) { this._node = ''; this._description = ''; @@ -417,7 +434,9 @@ onnx.Graph = class { this._description = graph.doc_string || ''; context = new onnx.GraphContext(context, graph.node); - + + // model parameter assignment here! + // console.log(graph) for (const initializer of graph.initializer) { const tensor = context.tensor(initializer.name); tensor.initializer = new onnx.Tensor(context, initializer, 'Initializer'); @@ -453,7 +472,7 @@ onnx.Graph = class { }); new onnx.Inference(graph.node, graph.output); context.push(graph.node, graph.input, graph.output); - this._nodes = context.pop(); + this._nodes = context.pop(); // get context._nodes() #Line1727 for (const input of graph.input) { const argument = context.argument(input.name); if (!argument.initializer) { @@ -578,6 +597,7 @@ onnx.Node = class { this._inputs = inputs; this._outputs = outputs; this._attributes = attributes.map((attribute) => new onnx.Attribute(context, op_type, domain, attribute)); + // console.log(this._attributes) this._chain = []; const identifier = domain ? domain + '.' + op_type : op_type; switch (identifier) { @@ -622,7 +642,7 @@ onnx.Node = class { }; onnx.Attribute = class { - + // `context` here is GraphContext constructor(context, op_type, domain, attribute) { this._name = attribute.name; this._description = attribute.doc_string || ''; @@ -695,9 +715,16 @@ onnx.Attribute = class { default: throw new onnx.Error("Unknown attribute type '" + attribute.type + "'."); } + // console.log(attribute.type) + // console.log(this._value) + // console.log(this._type) + // see #L1294 GraphMetadata const metadata = context.metadata.attribute(op_type, domain, attribute.name); + // console.log(metadata) if (metadata) { + // console.log(Object.prototype.hasOwnProperty.call(metadata, 'default') && this._value == metadata.default) // false + // console.log(metadata.type === 'DataType') // false if (Object.prototype.hasOwnProperty.call(metadata, 'default') && this._value == metadata.default) { this._visible = false; } @@ -1503,7 +1530,7 @@ onnx.ModelContext = class { }; onnx.GraphContext = class { - + // context here means ModelContext constructor(context, nodes) { this._context = context; this._decoder = new TextDecoder('utf-8'); @@ -1570,9 +1597,13 @@ onnx.GraphContext = class { } tensor(name) { + // console.log(this._tensors) + // console.log(name) + if (!this._tensors.has(name)) { this._tensors.set(name, { name: name }); } + // console.log(this._tensors) return this._tensors.get(name); } @@ -1591,6 +1622,8 @@ onnx.GraphContext = class { argument(name) { if (!this._arguments.has(name)) { const tensor = this.tensor(name); + // console.log(name) + // console.log(tensor) const type = tensor.initializer ? tensor.initializer.type : tensor.type || null; this._arguments.set(name, new onnx.Argument(name, type, tensor.initializer, tensor.annotation, tensor.description)); } @@ -1691,6 +1724,9 @@ onnx.GraphContext = class { node.input.length === 0 && node.output.length === 1 && node.output[0] && inputMap.get(node.output[0].name) === 1 && outputMap.get(node.output[0].name) === 1; const attribute = constant ? node.attribute[0] : null; + // console.log(node) + // console.log(constant) // false + // console.log(attribute) // null if (attribute && attribute.name === 'value' && attribute.type === onnx.AttributeType.TENSOR && attribute.t) { const tensor = this.tensor(node.output[0].name); tensor.initializer = new onnx.Tensor(this, attribute.t, 'Constant'); @@ -1705,15 +1741,22 @@ onnx.GraphContext = class { }); for (let node of nodes) { const schema = this._context.metadata.type(node.op_type, node.domain); + // console.log(node) // NodeProto. It contains the uploaded model data + // console.log(schema) // get the corresponding schema of this node from Metadata const inputs = []; node.input = node.input || []; for (let i = 0; i < node.input.length; ) { const input = schema && schema.inputs && i < schema.inputs.length ? schema.inputs[i] : { name: i.toString() }; const count = input.list ? node.input.length - i : 1; + + // slice the equal length of list from the upload model node + // and convert them to Argument list + // (instantiate a node here) const list = node.input.slice(i, i + count).map((input) => this.argument(input.name)); inputs.push(new onnx.Parameter(input.name, list)); i += count; } + // console.log(inputs) const outputs = []; node.output = node.output || []; for (let i = 0; i < node.output.length; ) { @@ -1723,8 +1766,10 @@ onnx.GraphContext = class { outputs.push(new onnx.Parameter(output.name, list)); i += count; } + // console.log(node) node = new onnx.Node(this, node.op_type, node.domain, node.name, node.doc_string, node.attribute, inputs, outputs); this._nodes.push(node); + // console.log(node) // const path = (node.name || '').split('/'); // path.pop(); diff --git a/static/view-sidebar.js b/static/view-sidebar.js index e2b783e..7ef7f66 100644 --- a/static/view-sidebar.js +++ b/static/view-sidebar.js @@ -129,7 +129,6 @@ sidebar.NodeSidebar = class { this._node = node; this._modelNodeName = modelNodeName; this._elements = []; - this._renameAuxelements = [] // auxilary elements for input/output renaming this._attributes = []; this._inputs = []; this._outputs = []; @@ -185,7 +184,6 @@ sidebar.NodeSidebar = class { this._addHeader('Inputs'); for (const input of inputs) { this._addInput(input.name, input); // 这里的input.name是小白格前面的名称（不是方格内的） - this.add_rename_aux_element(input.arguments); } } @@ -194,7 +192,6 @@ sidebar.NodeSidebar = class { this._addHeader('Outputs'); for (const output of outputs) { this._addOutput(output.name, output); - this.add_rename_aux_element(output.arguments); } } @@ -210,6 +207,20 @@ sidebar.NodeSidebar = class { this.add_separator(this._elements, 'sidebar-view-separator'); this._addHeader('Rename helper'); + if (inputs && inputs.length > 0) { + for (const input of inputs) { + this.add_rename_aux_element(input.arguments); + } + } + if (outputs && outputs.length > 0) { + for (const output of outputs) { + this.add_rename_aux_element(output.arguments); + } + } + + this.add_separator(this._elements, 'sidebar-view-separator'); + this._addHeader('Add children node'); + this._addDropdownSelector('AddChildrenNode'); } @@ -247,10 +258,9 @@ sidebar.NodeSidebar = class { // console.log(this._host._view._graph._renameMap); }); - this._renameAuxelements.push(origNameElement); - this._renameAuxelements.push(newNameElement) - this.add_separator(this._renameAuxelements, 'sidebar-view-separator') - + this._elements.push(origNameElement); + this._elements.push(newNameElement); + } } } @@ -260,9 +270,7 @@ sidebar.NodeSidebar = class { render() { // console.log(this._elements) - // console.log(this._renameAuxelements) - // return this._elements; - return this._elements.concat(this._renameAuxelements); + return this._elements; } _addHeader(title) { @@ -333,6 +341,16 @@ sidebar.NodeSidebar = class { this._host._view._graph.reset_node(this._modelNodeName) }); } + } + + _addDropdownSelector() { + const selectorElement = this._host.document.createElement('SELECT'); + this._elements.push(selectorElement); + // console.log(this._host._view._model.supported_ops) + for (const op of this._host._view._model.supported_ops) { + var option = new Option(op, op); + selectorElement.appendChild(option); + } }