wangchunlin
/
onnx-modifier
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1123
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'e1ad382665'
${ noResults }
onnx-modifier/static/python.js

4090 lines
166 KiB
JavaScript
Raw Blame History Unescape Escape

This file contains invisible Unicode characters!

This file contains invisible Unicode characters that may be processed differently from what appears below. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to reveal hidden characters.

// Experimental Python parser
var python = python || {};
python.Parser = class {
constructor(text, file, debug) {
this._tokenizer = new python.Tokenizer(text, file);
this._debug = debug;
if (!python.Parser._precedence) {
python.Parser._precedence = {
'or': 2, 'and': 3, 'not' : 4,
'in': 5, 'instanceof': 5, 'is': 5, '<': 5, '>': 5, '<=': 5, '>=': 5, '<>': 5, '==': 5, '!=': 5,
'|': 6, '^' : 7, '&' : 8,
'<<': 9, '>>': 9, '+': 10, '-': 10, '*': 11, '@': 11, '/': 11, '//': 11, '%': 11,
// '+': 12, '-': 12,
'~': 13, '**': 14
};
}
}
parse() {
const node = this._node('program');
node.body = [];
while (!this._tokenizer.match('eof')) {
const statement = this._parseStatement();
if (statement) {
node.body.push(statement);
continue;
}
if (this._tokenizer.eat('\n') || this._tokenizer.eat(';') || this._tokenizer.peek().type == 'eof') {
continue;
}
if (this._tokenizer.eat('indent') && this._tokenizer.peek().type == 'eof') {
continue;
}
throw new python.Error('Unknown statement' + this._tokenizer.location());
}
return node;
}
_parseSuite() {
const node = this._node('block');
node.statements = [];
let statement = null;
if (this._tokenizer.eat('\n')) {
if (this._tokenizer.eat('indent')) {
while (!this._tokenizer.eat('eof') && !this._tokenizer.eat('dedent')) {
if (this._tokenizer.eat(';')) {
continue;
}
statement = this._parseStatement();
if (statement) {
node.statements.push(statement);
continue;
}
if (this._tokenizer.eat('\n')) {
continue;
}
if (this._tokenizer.match('dedent') || this._tokenizer.match('eof')) {
continue;
}
throw new python.Error('Empty statement' + this._tokenizer.location());
}
}
}
else if (!this._tokenizer.eat('eof')) {
while (!this._tokenizer.match('\n') && !this._tokenizer.match('eof') && !this._tokenizer.match('dedent')) {
if (this._tokenizer.eat(';')) {
continue;
}
statement = this._parseStatement();
if (statement) {
node.statements.push(statement);
continue;
}
throw new python.Error('Empty statement' + this._tokenizer.location());
}
this._tokenizer.eat('\n');
}
return node;
}
_parseStatement() {
let node = this._node();
node = this._eat('id', 'break');
if (node) {
return node;
}
node = this._eat('id', 'continue');
if (node) {
return node;
}
node = this._eat('id', 'return');
if (node) {
node.expression = this._parseExpression(-1, [], true);
return node;
}
node = this._eat('id', 'raise');
if (node) {
node.exception = this._parseExpression(-1, [ 'from' ]);
if (this._tokenizer.eat('id', 'from')) {
node.from = this._parseExpression();
}
else if (this._tokenizer.eat(',')) {
node.exception = [ node.exception ];
node.exception.push(this._parseExpression());
if (this._tokenizer.eat(',')) {
node.exception.push(this._parseExpression());
}
}
return node;
}
node = this._eat('id', 'assert');
if (node) {
node.condition = this._parseExpression();
while (this._tokenizer.eat(',')) {
node.condition = { type: 'list', value: [ node.condition ] };
node.condition.value.push(this._parseExpression());
}
return node;
}
node = this._eat('id', 'exec');
if (node) {
node.variable = this._parseExpression(-1, [ 'in' ]);
if (this._tokenizer.eat('in')) {
do {
node.target = node.target || [];
node.target.push(this._parseExpression(-1, [ 'in' ], false));
}
while (this._tokenizer.eat(','));
}
return node;
}
node = this._eat('id', 'global');
if (node) {
node.variable = [];
do {
node.variable.push(this._parseName());
}
while (this._tokenizer.eat(','));
return node;
}
node = this._eat('id', 'nonlocal');
if (node) {
node.variable = [];
do {
node.variable.push(this._parseName());
}
while (this._tokenizer.eat(','));
return node;
}
node = this._eat('id', 'import');
if (node) {
node.modules = [];
do {
const module = this._node('module');
module.name = this._parseExpression(-1, [], false);
if (this._tokenizer.eat('id', 'as')) {
module.as = this._parseExpression(-1, [], false);
}
node.modules.push(module);
}
while (this._tokenizer.eat(','));
return node;
}
node = this._eat('id', 'from');
if (node) {
const dots = this._tokenizer.peek();
if (dots && Array.from(dots.type).every((c) => c == '.')) {
node.from = this._eat(dots.type);
node.from.expression = this._parseExpression();
}
else {
node.from = this._parseExpression();
}
this._tokenizer.expect('id', 'import');
node.import = [];
const close = this._tokenizer.eat('(');
do {
const symbol = this._node();
symbol.symbol = this._parseExpression(-1, [], false);
if (this._tokenizer.eat('id', 'as')) {
symbol.as = this._parseExpression(-1, [], false);
}
node.import.push(symbol);
}
while (this._tokenizer.eat(','));
if (close) {
this._tokenizer.expect(')');
}
return node;
}
node = this._eat('id', 'class');
if (node) {
node.name = this._parseName().value;
if (this._tokenizer.peek().value === '(') {
node.base = this._parseArguments();
}
this._tokenizer.expect(':');
node.body = this._parseSuite();
return node;
}
const async = this._eat('id', 'async');
if (async &&
!this._tokenizer.match('id', 'def') &&
!this._tokenizer.match('id', 'with') &&
!this._tokenizer.match('id', 'for')) {
throw new python.Error("Expected 'def', 'with' or 'for'" + this._tokenizer.location());
}
node = this._eat('id', 'def');
if (node) {
if (async) {
node.async = async;
}
node.name = this._parseName().value;
this._tokenizer.expect('(');
node.parameters = this._parseParameters(')');
if (this._tokenizer.eat('->')) {
node.returnType = this._parseType();
}
this._tokenizer.expect(':');
node.body = this._parseSuite();
return node;
}
node = this._eat('id', 'del');
if (node) {
node.expression = this._parseExpression(-1, [], true);
return node;
}
node = this._eat('id', 'print');
if (node) {
node.expression = this._parseExpression(-1, [], true);
return node;
}
node = this._eat('id', 'if');
if (node) {
node.condition = this._parseExpression();
this._tokenizer.expect(':');
node.then = this._parseSuite();
let current = node;
this._tokenizer.eat('\n');
while (this._tokenizer.eat('id', 'elif')) {
current.else = this._node('if');
current = current.else;
current.condition = this._parseExpression();
this._tokenizer.expect(':');
current.then = this._parseSuite();
this._tokenizer.eat('\n');
}
if (this._tokenizer.eat('id', 'else')) {
this._tokenizer.expect(':');
current.else = this._parseSuite();
}
return node;
}
node = this._eat('id', 'while');
if (node) {
node.condition = this._parseExpression();
this._tokenizer.expect(':');
node.body = this._parseSuite();
if (this._tokenizer.eat('id', 'else')) {
this._tokenizer.expect(':');
node.else = this._parseSuite();
}
return node;
}
node = this._eat('id', 'pass');
if (node) {
return node;
}
node = this._eat('id', 'for');
if (node) {
node.variable = [];
node.variable.push(this._parseExpression(-1, [ 'in' ]));
while (this._tokenizer.eat(',')) {
if (this._tokenizer.match('id', 'in')) {
node.variable.push({});
break;
}
node.variable.push(this._parseExpression(-1, [ 'in' ]));
}
this._tokenizer.expect('id', 'in');
node.target = [];
node.target.push(this._parseExpression());
while (this._tokenizer.eat(',')) {
if (this._tokenizer.match(':')) {
node.target.push({});
break;
}
node.target.push(this._parseExpression(-1, [ 'in' ]));
}
this._tokenizer.expect(':');
node.body = this._parseSuite();
if (this._tokenizer.eat('id', 'else')) {
this._tokenizer.expect(':');
node.else = this._parseSuite();
}
return node;
}
node = this._eat('id', 'with');
if (node) {
if (async) {
node.async = async;
}
node.item = [];
do {
const item = this._node();
item.type = 'with_item';
item.expression = this._parseExpression();
if (this._tokenizer.eat('id', 'as')) {
item.variable = this._parseExpression();
}
node.item.push(item);
}
while (this._tokenizer.eat(','));
this._tokenizer.expect(':');
node.body = this._parseSuite();
return node;
}
node = this._eat('id', 'try');
if (node) {
this._tokenizer.expect(':');
node.body = this._parseSuite();
node.except = [];
while (this._tokenizer.match('id', 'except')) {
const except = this._node('except');
this._tokenizer.expect('id', 'except');
except.clause = [];
except.clause.push(this._parseExpression());
while (this._tokenizer.eat(',')) {
if (this._tokenizer.match(':') || this._tokenizer.match('as')) {
except.clause.push({});
break;
}
except.clause.push(this._parseExpression());
}
if (this._tokenizer.eat('id', 'as')) {
except.variable = this._parseExpression();
}
this._tokenizer.expect(':');
except.body = this._parseSuite();
node.except.push(except);
}
if (this._tokenizer.match('id', 'else')) {
node.else = this._node('else');
this._tokenizer.expect('id', 'else');
this._tokenizer.expect(':');
node.else.body = this._parseSuite();
}
if (this._tokenizer.match('id', 'finally')) {
node.finally = this._node('finally');
this._tokenizer.expect('id', 'finally');
this._tokenizer.expect(':');
node.finally.body = this._parseSuite();
}
return node;
}
if (this._tokenizer.match('@')) {
node = this._node('decorator');
this._tokenizer.expect('@');
node.value = this._parseExpression();
if (!node.value || (node.value.type !== 'call' && node.value.type !== 'id' && node.value.type !== '.')) {
throw new python.Error('Invalid decorator' + this._tokenizer.location());
}
return node;
}
const expression = this._parseExpression(-1, [], true);
if (expression) {
if (expression.type == 'id' && this._tokenizer.eat(':')) {
node = this._node('var');
node.name = expression.value;
node.location = expression.location;
node.variableType = this._parseExpression(-1, [ '=' ]);
if (this._tokenizer.eat('=')) {
node.initializer = this._parseExpression();
}
return node;
}
let statement = false;
switch (expression.type) {
case '=':
case ':=':
case '==':
case '!=':
case '+=':
case '-=':
case '*=':
case '@=':
case '/=':
case '//=':
case '**=':
case '&=':
case '|=':
case '%=':
case '>>=':
case '<<=':
case '>>':
case '<<':
case '>=':
case '<=':
case '<':
case '>':
case '%':
case '^=':
case '...':
case 'call':
case 'assert':
case 'raise':
case 'string':
case 'list':
case 'var':
case '.':
case '[]':
case 'yield':
case '+':
case '-':
case '*':
case '**':
case '@':
case '/':
case '//':
case '~':
case '&':
case '^':
case '|':
case 'not':
case 'id':
case 'number':
case 'in':
case 'and':
case 'or':
case 'if':
case 'for':
case 'tuple':
case 'lambda':
case 'await':
statement = true;
break;
}
if (statement) {
return expression;
}
throw new python.Error("Unhandled expression" + this._tokenizer.location());
}
return null;
}
_parseExpression(minPrecedence, terminal, tuple) {
minPrecedence = minPrecedence || -1;
const terminalSet = new Set(terminal);
const stack = [];
for (;;) {
let node = this._node();
const token = this._tokenizer.peek();
if (stack.length == 1 && terminalSet.has(token.value)) {
break;
}
const precedence = python.Parser._precedence[token.value];
if (precedence) {
if (precedence >= minPrecedence) {
this._tokenizer.read();
node.type = token.value;
if (token.type == 'id' && (token.value === 'in' || token.value === 'not')) {
if (token.value === 'in') {
node.type = 'in';
}
else if (this._tokenizer.eat('id', 'in')) {
node.type = 'not in';
}
else {
node.type = 'not';
node.expression = this._parseExpression(precedence, terminal, tuple === false ? false : true);
stack.push(node);
continue;
}
}
else if (token.value == '~') {
node.type = '~';
node.expression = this._parseExpression(precedence, terminal, tuple === false ? false : true);
stack.push(node);
continue;
}
else if (token.type == 'id' && token.value == 'is') {
if (this._tokenizer.eat('id', 'not')) {
node.type = 'is not';
}
}
node.left = stack.pop();
node.right = this._parseExpression(precedence, terminal, tuple === false ? false : true);
stack.push(node);
continue;
}
}
if (this._tokenizer.eat(':=')) {
node.type = ':=';
node.target = stack.pop();
node.expression = this._parseExpression(-1, terminal, tuple === false ? false : true);
stack.push(node);
continue;
}
if (this._tokenizer.eat('=')) {
node.type = '=';
node.target = stack.pop();
node.expression = this._parseExpression(-1, terminal, tuple === false ? false : true);
stack.push(node);
continue;
}
switch (token.type) {
case '-=':
case '**=':
case '*=':
case '//=':
case '/=':
case '&=':
case '%=':
case '^=':
case '+=':
case '<<=':
case '>>=':
case '|=':
case '@=':
node = this._node(token.type);
this._tokenizer.expect(token.type);
node.target = stack.pop();
node.expression = this._parseExpression(-1, terminal, true);
stack.push(node);
continue;
}
node = this._eat('id', 'if');
if (node) {
node.then = stack.pop();
node.condition = this._parseExpression();
this._tokenizer.expect('id', 'else');
node.else = this._parseExpression();
stack.push(node);
continue;
}
while (this._tokenizer.match('id', 'for') || this._tokenizer.match('id', 'async')) {
const async = this._eat('id', 'async');
if (async && !this._tokenizer.match('id', 'for')) {
throw new python.Error("Expected 'for'" + this._tokenizer.location());
}
node = this._eat('id', 'for');
if (node) {
if (async) {
node.async = async;
}
node.expression = stack.pop();
node.variable = this._parseExpression(-1, [ 'in' ], true);
this._tokenizer.expect('id', 'in');
node.target = this._parseExpression(-1, [ 'for', 'if' ], true);
while (this._tokenizer.eat('id', 'if')) {
node.condition = node.condition || [];
node.condition.push(this._parseExpression(-1, [ 'for', 'if' ]));
}
stack.push(node);
}
}
node = this._eat('id', 'lambda');
if (node) {
node.parameters = this._parseParameters(':');
node.body = this._parseExpression(-1, terminal, false);
stack.push(node);
continue;
}
node = this._eat('id', 'yield');
if (node) {
if (this._tokenizer.eat('id', 'from')) {
node.from = this._parseExpression(-1, [], true);
}
else {
node.expression = [];
do {
node.expression.push(this._parseExpression(-1, [], false));
}
while (this._tokenizer.eat(','));
}
stack.push(node);
continue;
}
node = this._eat('id', 'await');
if (node) {
node.expression = this._parseExpression(minPrecedence, terminal, tuple);
stack.push(node);
continue;
}
node = this._eat('.');
if (node) {
this._tokenizer.eat('\n');
node.target = stack.pop();
node.member = this._parseName();
stack.push(node);
continue;
}
if (this._tokenizer.peek().value === '(') {
if (stack.length == 0) {
node = this._node('tuple');
const args = this._parseArguments();
if (args.length == 1) {
stack.push(args[0]);
}
else {
node.value = args;
stack.push(node);
}
}
else {
node = this._node('call');
node.target = stack.pop();
node.arguments = this._parseArguments();
stack.push(node);
}
continue;
}
if (this._tokenizer.peek().value === '[') {
if (stack.length == 0) {
stack.push(this._parseExpressions());
}
else {
node = this._node('[]');
node.target = stack.pop();
node.arguments = this._parseSlice();
stack.push(node);
}
continue;
}
if (this._tokenizer.peek().value == '{') {
stack.push(this._parseDictOrSetMaker());
continue;
}
node = this._node();
const literal = this._parseLiteral();
if (literal) {
if (stack.length > 0 && literal.type == 'number' &&
(literal.value.startsWith('-') || literal.value.startsWith('+'))) {
node.type = literal.value.substring(0, 1);
literal.value = literal.value.substring(1);
node.left = stack.pop();
node.right = literal;
stack.push(node);
}
else if (stack.length == 1 && literal.type == 'string' && stack[0].type == 'string') {
stack[0].value += literal.value;
}
else {
if (literal.type === 'number') {
switch (literal.value) {
case 'inf': literal.value = Infinity; break;
case '-inf': literal.value = -Infinity; break;
}
}
stack.push(literal);
}
continue;
}
if (this._tokenizer.peek().keyword) {
break;
}
node = this._eat('...');
if (node) {
stack.push(node);
continue;
}
const identifier = this._parseName();
if (identifier) {
stack.push(identifier);
continue;
}
if (tuple === true && stack.length == 1 && this._tokenizer.eat(',')) {
if (stack[0].type === 'tuple') {
node = stack[0];
}
else {
node = this._node('tuple');
node.value = [ stack.pop() ];
stack.push(node);
}
// for, bar, = <expr>
if (this._tokenizer.peek().value === '=') {
continue;
}
if (!this._tokenizer.match('=') && !terminalSet.has(this._tokenizer.peek().value)) {
const nextTerminal = terminal.slice(0).concat([ ',', '=' ]);
const expression = this._parseExpression(minPrecedence, nextTerminal, tuple);
if (expression) {
node.value.push(expression);
continue;
}
}
break;
}
break;
}
if (stack.length == 1) {
return stack.pop();
}
if (stack.length != 0) {
throw new python.Error('Unexpected expression' + this._tokenizer.location());
}
return null;
}
_parseDictOrSetMaker() {
const list = [];
this._tokenizer.expect('{');
let dict = true;
while (!this._tokenizer.eat('}')) {
const item = this._parseExpression(-1, [], false);
if (item == null) {
throw new python.Error('Expected expression' + this._tokenizer.location());
}
if (!this._tokenizer.eat(':')) {
dict = false;
}
if (dict) {
const value = this._parseExpression(-1, [], false);
if (value == null) {
throw new python.Error('Expected expression' + this._tokenizer.location());
}
list.push({ type: 'pair', key: item, value: value });
}
else {
list.push(item);
}
this._tokenizer.eat(',');
this._tokenizer.eat('\n');
if (this._tokenizer.eat('}')) {
break;
}
}
if (dict) {
return { type: 'dict', value: list };
}
return { type: 'set', value: list };
}
_parseExpressions() {
const list = [];
this._tokenizer.expect('[');
while (!this._tokenizer.eat(']')) {
const expression = this._parseExpression();
if (expression == null) {
throw new python.Error('Expected expression' + this._tokenizer.location());
}
list.push(expression);
this._tokenizer.eat(',');
while (this._tokenizer.eat('\n')) {
// continue
}
if (this._tokenizer.eat(']')) {
break;
}
}
return { type: 'list', value: list };
}
_parseSlice() {
let node = { type: '::' };
let list = [];
const group = [ 'start', 'stop', 'step' ];
this._tokenizer.expect('[');
while (!this._tokenizer.eat(']')) {
if (this._tokenizer.eat(':')) {
node[group.shift()] = { type: 'list', value: list };
list = [];
continue;
}
if (this._tokenizer.eat(',')) {
// list.push({});
continue;
}
if (this._tokenizer.peek().value != ']') {
const expression = this._parseExpression();
if (expression == null) {
throw new python.Error('Expected expression' + this._tokenizer.location());
}
list.push(expression);
}
}
if (list.length > 0) {
node[group.shift()] = { type: 'list', value: list };
}
if (node.start && !node.stop && !node.step) {
node = node.start;
}
return node;
}
_parseName() {
const token = this._tokenizer.peek();
if (token.type == 'id' && !token.keyword) {
this._tokenizer.read();
return token;
}
return null;
}
_parseLiteral() {
const token = this._tokenizer.peek();
if (token.type == 'string' || token.type == 'number' || token.type == 'boolean') {
this._tokenizer.read();
return token;
}
return null;
}
_parseTypeArguments() {
const list = [];
this._tokenizer.expect('[');
while (!this._tokenizer.eat(']')) {
const type = this._parseType();
if (type == null) {
throw new python.Error('Expected type ' + this._tokenizer.location());
}
list.push(type);
if (!this._tokenizer.eat(',')) {
this._tokenizer.expect(']');
break;
}
}
return list;
}
_parseType() {
const type = this._node();
type.type = 'type';
type.name = this._parseExpression(-1, [ '[', '=' ]);
if (type.name) {
if (this._tokenizer.peek().value === '[') {
type.arguments = this._parseTypeArguments();
}
return type;
}
return null;
}
_parseParameter(terminal) {
const node = this._node('parameter');
if (this._tokenizer.eat('/')) {
node.name = '/';
return node;
}
if (this._tokenizer.eat('**')) {
node.parameterType = '**';
}
if (this._tokenizer.eat('*')) {
node.parameterType = '*';
}
const identifier = this._parseName();
if (identifier !== null) {
node.name = identifier.value;
if (terminal !== ':' && this._tokenizer.eat(':')) {
node.parameterType = this._parseType();
}
if (this._tokenizer.eat('=')) {
node.initializer = this._parseExpression();
}
return node;
}
return null;
}
_parseParameters(terminal) {
const list = [];
while (!this._tokenizer.eat(terminal)) {
this._tokenizer.eat('\n');
if (this._tokenizer.eat('(')) {
list.push(this._parseParameters(')'));
}
else {
list.push(this._parseParameter(terminal));
}
this._tokenizer.eat('\n');
if (!this._tokenizer.eat(',')) {
this._tokenizer.expect(terminal);
break;
}
}
return list;
}
_parseArguments() {
const list = [];
this._tokenizer.expect('(');
while (!this._tokenizer.eat(')')) {
if (this._tokenizer.eat('\n')) {
continue;
}
const expression = this._parseExpression(-1, [], false);
if (expression == null) {
throw new python.Error('Expected expression ' + this._tokenizer.location());
}
list.push(expression);
if (!this._tokenizer.eat(',')) {
this._tokenizer.eat('\n');
this._tokenizer.expect(')');
break;
}
}
return list;
}
_node(type) {
const node = {};
node.location = this._tokenizer.location();
if (type) {
node.type = type;
}
return node;
}
_eat(type, value) {
if (this._tokenizer.match(type, value)) {
const node = this._node(type === 'id' ? value : type);
this._tokenizer.expect(type, value);
return node;
}
return null;
}
};
python.Tokenizer = class {
constructor(text, file) {
this._text = text;
this._file = file;
this._position = 0;
this._lineStart = 0;
this._line = 0;
this._token = { type: '', value: '' };
this._brackets = 0;
this._indentation = [];
this._outdent = 0;
if (!python.Tokenizer._whitespace) {
python.Tokenizer._whitespace = new RegExp('[\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff]');
const identifierStartChars = '\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376\u0377\u037a-\u037d\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0620-\u064a\u066e\u066f\u0671-\u06d3\u06d5\u06e5\u06e6\u06ee\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097f\u0985-\u098c\u098f\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc\u09dd\u09df-\u09e1\u09f0\u09f1\u0a05-\u0a0a\u0a0f\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32\u0a33\u0a35\u0a36\u0a38\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0\u0ae1\u0b05-\u0b0c\u0b0f\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32\u0b33\u0b35-\u0b39\u0b3d\u0b5c\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99\u0b9a\u0b9c\u0b9e\u0b9f\u0ba3\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58\u0c59\u0c60\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0\u0ce1\u0cf1\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e33\u0e40-\u0e46\u0e81\u0e82\u0e84\u0e87\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa\u0eab\u0ead-\u0eb0\u0eb2\u0eb3\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u212f-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2e2f\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a\ua62b\ua640-\ua66e\ua67f-\ua697\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40\ufb41\ufb43\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc';
const identifierChars = '\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610-\u061a\u0620-\u0649\u0672-\u06d3\u06e7-\u06e8\u06fb-\u06fc\u0730-\u074a\u0800-\u0814\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0840-\u0857\u08e4-\u08fe\u0900-\u0903\u093a-\u093c\u093e-\u094f\u0951-\u0957\u0962-\u0963\u0966-\u096f\u0981-\u0983\u09bc\u09be-\u09c4\u09c7\u09c8\u09d7\u09df-\u09e0\u0a01-\u0a03\u0a3c\u0a3e-\u0a42\u0a47\u0a48\u0a4b-\u0a4d\u0a51\u0a66-\u0a71\u0a75\u0a81-\u0a83\u0abc\u0abe-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ae2-\u0ae3\u0ae6-\u0aef\u0b01-\u0b03\u0b3c\u0b3e-\u0b44\u0b47\u0b48\u0b4b-\u0b4d\u0b56\u0b57\u0b5f-\u0b60\u0b66-\u0b6f\u0b82\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd7\u0be6-\u0bef\u0c01-\u0c03\u0c46-\u0c48\u0c4a-\u0c4d\u0c55\u0c56\u0c62-\u0c63\u0c66-\u0c6f\u0c82\u0c83\u0cbc\u0cbe-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5\u0cd6\u0ce2-\u0ce3\u0ce6-\u0cef\u0d02\u0d03\u0d46-\u0d48\u0d57\u0d62-\u0d63\u0d66-\u0d6f\u0d82\u0d83\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0df2\u0df3\u0e34-\u0e3a\u0e40-\u0e45\u0e50-\u0e59\u0eb4-\u0eb9\u0ec8-\u0ecd\u0ed0-\u0ed9\u0f18\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f41-\u0f47\u0f71-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1029\u1040-\u1049\u1067-\u106d\u1071-\u1074\u1082-\u108d\u108f-\u109d\u135d-\u135f\u170e-\u1710\u1720-\u1730\u1740-\u1750\u1772\u1773\u1780-\u17b2\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1920-\u192b\u1930-\u193b\u1951-\u196d\u19b0-\u19c0\u19c8-\u19c9\u19d0-\u19d9\u1a00-\u1a15\u1a20-\u1a53\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1b46-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1bb0-\u1bb9\u1be6-\u1bf3\u1c00-\u1c22\u1c40-\u1c49\u1c5b-\u1c7d\u1cd0-\u1cd2\u1d00-\u1dbe\u1e01-\u1f15\u200c\u200d\u203f\u2040\u2054\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2d81-\u2d96\u2de0-\u2dff\u3021-\u3028\u3099\u309a\ua640-\ua66d\ua674-\ua67d\ua69f\ua6f0-\ua6f1\ua7f8-\ua800\ua806\ua80b\ua823-\ua827\ua880-\ua881\ua8b4-\ua8c4\ua8d0-\ua8d9\ua8f3-\ua8f7\ua900-\ua909\ua926-\ua92d\ua930-\ua945\ua980-\ua983\ua9b3-\ua9c0\uaa00-\uaa27\uaa40-\uaa41\uaa4c-\uaa4d\uaa50-\uaa59\uaa7b\uaae0-\uaae9\uaaf2-\uaaf3\uabc0-\uabe1\uabec\uabed\uabf0-\uabf9\ufb20-\ufb28\ufe00-\ufe0f\ufe20-\ufe26\ufe33\ufe34\ufe4d-\ufe4f\uff10-\uff19\uff3f';
python.Tokenizer._identifierStart = new RegExp('[' + identifierStartChars + ']');
/* eslint-disable */
python.Tokenizer._identifierChar = new RegExp('[' + identifierStartChars + identifierChars + ']');
/* eslint-enable */
}
}
peek() {
if (!this._cache) {
this._token = this._tokenize(this._token);
this._cache = true;
}
return this._token;
}
read() {
if (!this._cache) {
this._token = this._tokenize(this._token);
}
const next = this._position + this._token.value.length;
while (this._position < next) {
if (python.Tokenizer._isNewline(this._get(this._position))) {
this._position = this._newLine(this._position);
this._lineStart = this._position;
this._line++;
}
else {
this._position++;
}
}
this._cache = false;
return this._token;
}
match(type, value) {
const token = this.peek();
if (token.type === type && (!value || token.value === value)) {
return true;
}
return false;
}
eat(type, value) {
const token = this.peek();
if (token.type === type && (!value || token.value === value)) {
this.read();
return true;
}
return false;
}
expect(type, value) {
const token = this.peek();
if (token.type !== type) {
throw new python.Error("Unexpected '" + token.value + "' instead of '" + type + "'" + this.location());
}
if (value && token.value !== value) {
throw new python.Error("Unexpected '" + token.value + "' instead of '" + value + "'" + this.location());
}
this.read();
}
location() {
return ' at ' + this._file + ':' + (this._line + 1).toString() + ':' + (this._position - this._lineStart + 1).toString();
}
static _isSpace(c) {
switch (c) {
case ' ':
case '\t':
case '\v': // 11
case '\f': // 12
case '\xA0': // 160
return true;
default:
if (c.charCodeAt(0) >= 0x1680) {
return python.Tokenizer._whitespace.test(c);
}
return false;
}
}
static _isNewline(c) {
switch(c) {
case '\n':
case '\r':
case '\u2028': // 8232
case '\u2029': // 8233
return true;
}
return false;
}
static _isIdentifierStartChar(c) {
if (c < 'A') {
return c === '$';
}
if (c <= 'Z') {
return true;
}
if (c < 'a') {
return c === '_';
}
if (c <= 'z') {
return true;
}
const code = c.charCodeAt(0);
if (code >= 0xAA) {
return python.Tokenizer._identifierStart.test(c);
}
return false;
}
static _isIdentifierChar(c) {
if (c < '0') {
return c === '$';
}
if (c <= '9') {
return true;
}
if (c < 'A') {
return false;
}
if (c <= 'Z') {
return true;
}
if (c < 'a') {
return c === '_';
}
if (c <= 'z') {
return true;
}
const code = c.charCodeAt(0);
if (code >= 0xAA) {
return python.Tokenizer._identifierChar.test(c);
}
return false;
}
static _isDecimal(c) {
return c >= '0' && c <= '9' || c === '_';
}
static _isHex(c) {
return python.Tokenizer._isDecimal(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || c === '_';
}
static _isOctal(c) {
return c >= '0' && c <= '7' || c === '_';
}
static _isBinary(c) {
return c === '0' || c === '1' || c === '_';
}
_get(position) {
return position >= this._text.length ? '\0' : this._text[position];
}
_skipLine() {
while (this._position < this._text.length) {
if (python.Tokenizer._isNewline(this._get(this._position))) {
break;
}
this._position++;
}
}
_skipWhitespace() {
while (this._position < this._text.length) {
const c = this._text[this._position];
if (c == '#') {
this._skipLine();
}
else if (python.Tokenizer._isSpace(c)) {
this._position++;
}
else if (c == '\\') {
// Explicit Line Continuation
this._position++;
if (python.Tokenizer._isNewline(this._get(this._position))) {
this._position = this._newLine(this._position);
this._lineStart = this._position;
this._line++;
}
else {
throw new python.Error("Unexpected '" + this._text[this._position] + "' after line continuation" + this.location());
}
}
else if (this._brackets > 0 && python.Tokenizer._isNewline(c)) {
// Implicit Line Continuation
this._position = this._newLine(this._position);
this._lineStart = this._position;
this._line++;
}
else {
break;
}
}
}
_newLine(position) {
if ((this._get(position) === '\n' && this._get(position + 1) === '\r') ||
(this._get(position) === '\r' && this._get(position + 1) === '\n')) {
return position + 2;
}
return position + 1;
}
_tokenize(token) {
if (this._token.type !== '\n') {
this._skipWhitespace();
}
if (this._token.type === 'dedent') {
this._indentation.pop();
this._outdent--;
if (this._outdent > 0) {
return { type: 'dedent', value: '' };
}
}
if (token.type == '\n') {
let indent = '';
let i = this._position;
while (i < this._text.length) {
const c = this._text[i];
if (python.Tokenizer._isSpace(c)) {
indent += c;
i++;
}
else if (python.Tokenizer._isNewline(c)) {
indent = '';
i = this._newLine(i);
this._position = i;
this._lineStart = i;
this._line++;
}
else if (c == '#') {
indent = '';
while (i < this._text.length && !python.Tokenizer._isNewline(this._text[i])) {
i++;
}
continue;
}
else {
break;
}
}
let type = null;
if (indent.length > 0) {
const current = this._indentation.length > 0 ? this._indentation[this._indentation.length - 1] : '';
if (indent.length > current.length) {
type = 'indent';
this._indentation.push(indent);
}
else if (indent.length > 0 && indent.length < current.length) {
type = 'dedent';
this._outdent = 0;
for (let j = this._indentation.length - 1; j >= 0 && indent.length < this._indentation[j].length; j--) {
this._outdent++;
}
}
else {
this._position += indent.length;
}
}
else if (i >= this._text.length) {
return { type: 'eof', value: '' };
}
else if (this._indentation.length > 0) {
type = 'dedent';
this._outdent = this._indentation.length;
}
switch (type) {
case 'indent':
case 'dedent':
return { type: type, value: indent };
}
}
if (this._position >= this._text.length) {
return { type: 'eof', value: '' };
}
const c = this._get(this._position);
const string = this._string();
if (string) {
return string;
}
switch (c) {
case '(':
case '[':
case '{':
this._brackets++;
return { type: c, value: c };
case ')':
case ']':
case '}':
if (this._brackets === 0) {
throw new python.Error("Unexpected '" + c + "'" + this.location);
}
this._brackets--;
return { type: c, value: c };
case ',':
case ';':
case '?':
return { type: c, value: c };
default: {
const number = this._number();
if (number) {
return number;
}
if (c === '.') {
let end = this._position + 1;
while (this._get(end) === '.') {
end++;
}
const text = this._text.substring(this._position, end);
return { type: text, value: text };
}
const identifier = this._identifier();
if (identifier) {
return identifier;
}
const operator = this._operator();
if (operator) {
return operator;
}
break;
}
}
if (c === '.') {
return { type: c, value: c };
}
if (c === '\\') {
return { type: '\\', value: c };
}
if (python.Tokenizer._isNewline(c)) {
return { type: '\n', value: this._text.substring(this._position, this._newLine(this._position)) };
}
throw new python.Error("Unexpected token '" + c + "'" + this.location());
}
_number() {
let c = this._get(this._position);
const sign = (c === '-' || c === '+') ? 1 : 0;
let i = this._position + sign;
c = this._get(i);
if (c === '0') {
let radix = 0;
const n = this._get(i + 1);
if ((n === 'x' || n === 'X') && python.Tokenizer._isHex(this._get(i + 2))) {
i += 2;
while (python.Tokenizer._isHex(this._get(i))) {
i += 1;
}
if (this._get(i) === 'l' || this._get(i) === 'L') {
i += 1;
}
radix = 16;
}
else if ((n === 'b' || n === 'B') && python.Tokenizer._isBinary(this._get(i + 2))) {
i += 2;
while (python.Tokenizer._isBinary(this._get(i))) {
i++;
}
radix = 2;
}
else if ((n === 'o' || n === 'O') && python.Tokenizer._isOctal(this._get(i + 2))) {
i += 2;
while (python.Tokenizer._isOctal(this._get(i))) {
i++;
}
radix = 8;
}
else if (n >= '0' && n <= '7') {
i++;
while (python.Tokenizer._isOctal(this._get(i))) {
i += 1;
}
if (this._get(i) === 'l' || this._get(i) === 'L') {
i += 1;
}
radix = 8;
}
if (radix > 0 && this._get(i) !== '.') {
const radixText = this._text.substring(this._position, i);
const radixParseText = radixText.indexOf('_') !== -1 ? radixText.split('_').join('') : radixText;
if (!isNaN(parseInt(radixParseText, radix))) {
return { type: 'number', value: radixText };
}
}
}
i = this._position + sign;
let decimal = false;
if (this._get(i) >= '1' && this._get(i) <= '9') {
while (python.Tokenizer._isDecimal(this._get(i))) {
i++;
}
c = this._get(i).toLowerCase();
decimal = c !== '.' && c !== 'e';
}
if (this._get(i) === '0') {
i++;
c = this._get(i).toLowerCase();
decimal = !python.Tokenizer._isDecimal(c) && c !== '.' && c !== 'e' && c !== 'j';
}
if (decimal) {
if (this._get(i) === 'j' || this._get(i) === 'J' || this._get(i) === 'l' || this._get(i) === 'L') {
return { 'type': 'number', value: this._text.substring(this._position, i + 1) };
}
const intText = this._text.substring(this._position, i);
if (!isNaN(parseInt(intText, 10))) {
return { type: 'number', value: intText };
}
}
i = this._position + sign;
if ((this._get(i) >= '0' && this._get(i) <= '9') ||
(this._get(i) === '.' && this._get(i + 1) >= '0' && this._get(i + 1) <= '9')) {
while (python.Tokenizer._isDecimal(this._get(i))) {
i++;
}
if (this._get(i) === '.') {
i++;
}
while (python.Tokenizer._isDecimal(this._get(i))) {
i++;
}
if (i > (this._position + sign)) {
if (this._get(i) === 'e' || this._get(i) === 'E') {
i++;
if (this._get(i) == '-' || this._get(i) == '+') {
i++;
}
if (!python.Tokenizer._isDecimal(this._get(i))) {
i = this._position;
}
else {
while (python.Tokenizer._isDecimal(this._get(i))) {
i++;
}
}
}
else {
while (python.Tokenizer._isDecimal(this._get(i))) {
i++;
}
}
}
if (i > (this._position + sign)) {
if (this._get(i) === 'j' || this._get(i) === 'J') {
return { type: 'number', value: this._text.substring(this._position, i + 1) };
}
const floatText = this._text.substring(this._position, i);
const floatParseText = floatText.indexOf('_') != -1 ? floatText.split('_').join('') : floatText;
if (!isNaN(parseFloat(floatParseText))) {
return { type: 'number', value: floatText };
}
}
}
i = this._position + sign;
if (this._get(i) === 'i' && this._get(i + 1) === 'n' && this._get(i + 2) === 'f' && !python.Tokenizer._isIdentifierChar(this._get(i + 3))) {
return { type: 'number', value: this._text.substring(this._position, i + 3) };
}
return null;
}
_identifier() {
let i = this._position;
if (python.Tokenizer._isIdentifierStartChar(this._get(i))) {
i++;
while (python.Tokenizer._isIdentifierChar(this._get(i))) {
i++;
}
}
if (i > this._position) {
const text = this._text.substring(this._position, i);
return { type: 'id', value: text, keyword: python.Tokenizer._isKeyword(text) };
}
return null;
}
_operator() {
let length = 0;
const c0 = this._get(this._position);
const c1 = this._get(this._position + 1);
const c2 = this._get(this._position + 2);
switch (c0) {
case '+':
case '&':
case '|':
case '^':
case '=':
case '!':
case '%':
case '~':
length = c1 === '=' ? 2 : 1;
break;
case '-':
length = c1 === '=' || c1 === '>' ? 2 : 1;
break;
case '*':
if (c1 === '*') {
length = c2 === '=' ? 3 : 2;
}
else {
length = c1 === '=' ? 2 : 1;
}
break;
case '/':
if (c1 === '/') {
length = c2 === '=' ? 3 : 2;
}
else {
length = c1 === '=' ? 2 : 1;
}
break;
case '<':
if (c1 === '>') {
length = 2;
}
else if (c1 === '<') {
length = c2 === '=' ? 3 : 2;
}
else {
length = c1 === '=' ? 2 : 1;
}
break;
case '>':
if (c1 === '>') {
length = c2 === '=' ? 3 : 2;
}
else {
length = c1 === '=' ? 2 : 1;
}
break;
case '@':
length = c1 === '=' ? 2 : 1;
break;
case ':':
length = c1 === '=' ? 2 : 1;
}
if (length > 0) {
const text = this._text.substring(this._position, this._position + length);
return { type: text, value: text };
}
return null;
}
_string() {
let i = this._position;
let prefix = -1;
if (this._get(i) === "'" || this._get(i) === '"') {
prefix = '';
}
else if (this._get(i + 1) === "'" || this._get(i + 1) === '"') {
const c = this._get(i);
switch (c.toLowerCase()) {
case 'b':
case 'f':
case 'r':
case 'u':
prefix = c;
break;
}
}
else if (this._get(i + 2) === "'" || this._get(i + 2) === '"') {
const cc = this._text.substr(this._position, 2);
switch (cc.toLowerCase()) {
case 'br':
case 'fr':
case 'rb':
case 'rf':
case 'ur':
prefix = cc;
break;
}
}
if (prefix.length >= 0) {
i += prefix.length;
let quote = '';
let count = 0;
const q0 = this._get(i);
const q1 = this._get(i + 1);
const q2 = this._get(i + 2);
switch (q0) {
case "'":
quote = q0;
count = (q1 === "'" && q2 === "'") ? 3 : 1;
break;
case '"':
quote = q0;
count = (q1 === '"' && q2 === '"') ? 3 : 1;
}
i += count;
if (count == 1) {
while (i < this._text.length) {
if (this._text[i] === quote) {
return { type: 'string', value: this._text.substring(this._position, i + 1) };
}
else if (this._text[i] === '\\' &&
(this._get(i + 1) == quote || this._get(i + 1) == '\n' || this._get(i + 1) == '\\')) {
i += 2;
}
else if (this._text[i] === '\r' || this._text[i] === '\n') {
break;
}
else {
i++;
}
}
}
else if (count == 3) {
while (i < this._text.length) {
if (this._get(i) === quote && this._get(i + 1) === quote && this._get(i + 2) === quote) {
return { type: 'string', value: this._text.substring(this._position, i + 3) };
}
else if (this._get(i) === '\\' && this._get(i + 1) === quote) {
i += 2;
continue;
}
i++;
}
}
}
i = this._position;
if (this._get(i) === '`') {
i++;
while (i < this._text.length) {
if (this._text[i] === '`') {
return { type: 'string', value: this._text.substring(this._position, i + 1) };
}
i++;
}
}
return null;
}
static _isKeyword(value) {
switch (value) {
case 'and':
case 'as':
case 'else':
case 'for':
case 'if':
case 'import':
case 'in':
case 'is':
case 'not':
case 'or':
return true;
}
return false;
}
};
python.Execution = class {
constructor(sources, exceptionCallback) {
const self = this;
this._sources = sources || new Map();
this._exceptionCallback = exceptionCallback;
this._utf8Decoder = new TextDecoder('utf-8');
this._packages = new Map();
this._unknownNameMap = new Set();
this._context = new python.Execution.Context();
this._context.scope.builtins = {};
this._context.scope.builtins.type = { __module__: 'builtins', __name__: 'type' };
this._context.scope.builtins.type.__class__ = this._context.scope.builtins.type;
this._context.scope.builtins.module = { __module__: 'builtins', __name__: 'module', __class__: this._context.scope.builtins.type };
this._context.scope.builtins.module.__type__ = this._context.scope.builtins.module;
this.registerModule('__builtin__');
this.registerModule('_codecs');
this.registerModule('argparse');
this.registerModule('collections');
this.registerModule('copy_reg');
this.registerModule('cuml');
this.registerModule('gensim');
this.registerModule('io');
this.registerModule('joblib');
this.registerModule('keras');
this.registerModule('lightgbm');
this.registerModule('numpy');
this.registerModule('nolearn');
this.registerModule('sklearn');
this.registerModule('typing');
this.registerModule('xgboost');
const builtins = this._context.scope.builtins;
const numpy = this._context.scope.numpy;
const typing = this._context.scope.typing;
this.registerType('builtins.function', class {});
this.registerType('builtins.method', class {});
this.registerType('builtins.dict', class {});
this.registerType('builtins.list', class {});
this.registerType('builtins.bool', class {});
this.registerType('builtins.int', class {});
this.registerType('builtins.float', class {});
this.registerType('builtins.object', class {});
this.registerType('builtins.str', class {});
this.registerType('builtins.tuple', class {});
this.registerType('typing._Final', class {});
this.registerType('typing._SpecialForm', class extends typing._Final {});
this.registerType('typing._BaseGenericAlias', class extends typing._Final {});
this.registerType('typing._GenericAlias', class extends typing._BaseGenericAlias {});
this.registerType('typing._SpecialGenericAlias', class extends typing._BaseGenericAlias {});
this.registerType('typing._TupleType', class extends typing._SpecialGenericAlias {});
typing.Optional = self.invoke('typing._SpecialForm', []);
typing.List = self.invoke('typing._SpecialGenericAlias', []);
typing.Dict = self.invoke('typing._SpecialGenericAlias', []);
typing.Tuple = self.invoke('typing._TupleType', []);
this.registerType('argparse.Namespace', class {
constructor(args) {
this.args = args;
}
});
this.registerType('collections.deque', class {
constructor(iterable) {
if (iterable) {
let i = 0;
for (const value of iterable) {
this[i++] = value;
}
this.length = i;
}
}
});
this.registerType('collections.OrderedDict', class extends Map {
constructor(items) {
super();
if (items) {
for (const pair of items) {
this.__setitem__(pair[0], pair[1]);
}
}
}
__setitem__(key, value) {
this.set(key, value);
}
});
this.registerType('cuml.common.array_descriptor.CumlArrayDescriptorMeta', class {});
this.registerType('cuml.ensemble.randomforestclassifier.RandomForestClassifier', class {});
this.registerType('cuml.raft.common.handle.Handle', class {
__setstate__(state) {
this._handle = state;
}
});
this.registerType('haiku._src.data_structures.FlatMapping', class {
constructor(dict) {
for (const key of Object.keys(dict)) {
this[key] = dict[key];
}
}
});
this.registerType('io.BytesIO', class {
constructor(buf, mode) {
this.mode = mode || 'r';
this._buf = this.mode === 'w' ? null : buf;
this._point = 0;
}
seek(offset) {
this._point = offset;
}
read(size) {
const start = this._point;
this._point = size !== undefined ? start + size : this._buf.length;
return this._buf.subarray(start, this._point);
}
write(data) {
const src = this._buf || new Uint8Array();
this._point = src.length + data.length;
this._buf = new Uint8Array(this._point);
this._buf.set(src, 0);
this._buf.set(data, src.length);
}
});
this.registerType('numpy.dtype', class {
constructor(obj, align, copy) {
switch (obj) {
case 'b1': case 'bool': this.name = 'bool'; this.itemsize = 1; this.kind = 'b'; break;
case 'i1': case 'int8': this.name = 'int8'; this.itemsize = 1; this.kind = 'i'; break;
case 'i2': case 'int16': this.name = 'int16'; this.itemsize = 2; this.kind = 'i'; break;
case 'i4': case 'int32': this.name = 'int32'; this.itemsize = 4; this.kind = 'i'; break;
case 'i8': case 'int64': case 'int': this.name = 'int64'; this.itemsize = 8; this.kind = 'i'; break;
case 'u1': case 'uint8': this.name = 'uint8'; this.itemsize = 1; this.kind = 'u'; break;
case 'u2': case 'uint16': this.name = 'uint16'; this.itemsize = 2; this.kind = 'u'; break;
case 'u4': case 'uint32': this.name = 'uint32'; this.itemsize = 4; this.kind = 'u'; break;
case 'u8': case 'uint64': case 'uint': this.name = 'uint64'; this.itemsize = 8; this.kind = 'u'; break;
case 'f2': case 'float16': this.name = 'float16'; this.itemsize = 2; this.kind = 'f'; break;
case 'f4': case 'float32': this.name = 'float32'; this.itemsize = 4; this.kind = 'f'; break;
case 'f8': case 'float64': case 'float': this.name = 'float64'; this.itemsize = 8; this.kind = 'f'; break;
case 'c8': case 'complex64': this.name = 'complex64'; this.itemsize = 8; this.kind = 'c'; break;
case 'c16': case 'complex128': case 'complex': this.name = 'complex128'; this.itemsize = 16; this.kind = 'c'; break;
default:
if (obj.startsWith('V')) {
this.itemsize = Number(obj.substring(1));
this.kind = 'V';
this.name = 'void' + (this.itemsize * 8).toString();
}
else if (obj.startsWith('O')) {
this.itemsize = Number(obj.substring(1));
this.kind = 'O';
this.name = 'object';
}
else if (obj.startsWith('S')) {
this.itemsize = Number(obj.substring(1));
this.kind = 'S';
this.name = 'string';
}
else if (obj.startsWith('U')) {
this.itemsize = Number(obj.substring(1));
this.kind = 'U';
this.name = 'string';
}
else if (obj.startsWith('M')) {
this.itemsize = Number(obj.substring(1));
this.kind = 'M';
this.name = 'datetime';
}
else {
throw new python.Error("Unknown dtype '" + obj.toString() + "'.");
}
break;
}
this.byteorder = '=';
if (align) {
this.align = align;
}
if (copy) {
this.copy = copy;
}
}
get str() {
return (this.byteorder === '=' ? '<' : this.byteorder) + this.kind + this.itemsize.toString();
}
__setstate__(state) {
switch (state.length) {
case 8:
this.version = state[0];
this.byteorder = state[1];
this.subarray = state[2];
this.names = state[3];
this.fields = state[4];
this.elsize = state[5];
this.alignment = state[6];
this.int_dtypeflags = state[7];
break;
case 9:
this.version = state[0];
this.byteorder = state[1];
this.subarray = state[2];
this.names = state[3];
this.fields = state[4];
this.elsize = state[5];
this.alignment = state[6];
this.int_dtypeflags = state[7];
this.metadata = state[8];
break;
default:
throw new python.Error("Unknown numpy.dtype setstate length '" + state.length.toString() + "'.");
}
}
});
this.registerType('gensim.models.doc2vec.Doctag', class {});
this.registerType('gensim.models.doc2vec.Doc2Vec', class {});
this.registerType('gensim.models.doc2vec.Doc2VecTrainables', class {});
this.registerType('gensim.models.doc2vec.Doc2VecVocab', class {});
this.registerType('gensim.models.fasttext.FastText', class {});
this.registerType('gensim.models.fasttext.FastTextTrainables', class {});
this.registerType('gensim.models.fasttext.FastTextVocab', class {});
this.registerType('gensim.models.fasttext.FastTextKeyedVectors', class {});
this.registerType('gensim.models.keyedvectors.Doc2VecKeyedVectors', class {});
this.registerType('gensim.models.keyedvectors.FastTextKeyedVectors', class {});
this.registerType('gensim.models.keyedvectors.KeyedVectors', class {});
this.registerType('gensim.models.keyedvectors.Vocab', class {});
this.registerType('gensim.models.keyedvectors.Word2VecKeyedVectors', class {});
this.registerType('gensim.models.phrases.Phrases', class {});
this.registerType('gensim.models.tfidfmodel.TfidfModel', class {});
this.registerType('gensim.models.word2vec.Vocab', class {});
this.registerType('gensim.models.word2vec.Word2Vec', class {});
this.registerType('gensim.models.word2vec.Word2VecTrainables', class {});
this.registerType('gensim.models.word2vec.Word2VecVocab', class {});
this.registerType('joblib.numpy_pickle.NumpyArrayWrapper', class {
constructor(/* subtype, shape, dtype */) {
}
__setstate__(state) {
this.subclass = state.subclass;
this.dtype = state.dtype;
this.shape = state.shape;
this.order = state.order;
this.allow_mmap = state.allow_mmap;
}
__read__(unpickler) {
if (this.dtype.name == 'object') {
return unpickler.load((name, args) => self.invoke(name, args), null);
}
else {
const size = this.dtype.itemsize * this.shape.reduce((a, b) => a * b, 1);
this.data = unpickler.read(size);
}
return self.invoke(this.subclass, [ this.shape, this.dtype, this.data ]);
}
});
this.registerType('keras.engine.sequential.Sequential', class {});
this.registerType('lightgbm.sklearn.LGBMRegressor', class {});
this.registerType('lightgbm.sklearn.LGBMClassifier', class {});
this.registerType('lightgbm.basic.Booster', class {
constructor() {
this.average_output = false;
this.models = [];
this.loaded_parameter = '';
}
__setstate__(state) {
if (typeof state.handle === 'string') {
this.LoadModelFromString(state.handle);
return;
}
Object.assign(this, state);
}
LoadModelFromString(model_str) {
const lines = model_str.split('\n');
const signature = lines.shift() || '?';
if (signature.trim() !== 'tree') {
throw new python.Error("Invalid signature '" + signature.trim() + "'.");
}
// GBDT::LoadModelFromString() in https://github.com/microsoft/LightGBM/blob/master/src/boosting/gbdt_model_text.cpp
const key_vals = new Map();
while (lines.length > 0 && !lines[0].startsWith('Tree=')) {
const cur_line = lines.shift().trim();
if (cur_line.length > 0) {
const strs = cur_line.split('=');
if (strs.length === 1) {
key_vals.set(strs[0], '');
}
else if (strs.length === 2) {
key_vals.set(strs[0], strs[1]);
}
else if (strs.length > 2) {
if (strs[0] === "feature_names") {
key_vals.set(strs[0], cur_line.substring("feature_names=".length));
}
else if (strs[0] == 'monotone_constraints') {
key_vals.set(strs[0], cur_line.substring('monotone_constraints='.length));
}
else {
throw new python.Error('Wrong line: ' + cur_line.substring(0, Math.min(128, cur_line.length)));
}
}
}
}
const atoi = (key, value) => {
if (key_vals.has(key)) {
return parseInt(key_vals.get(key), 10);
}
if (value !== undefined) {
return value;
}
throw new python.Error('Model file does not specify ' + key + '.');
};
const list = (key, size) => {
if (key_vals.has(key)) {
const value = key_vals.get(key).split(' ');
if (value.length !== size) {
throw new python.Error('Wrong size of ' + key + '.');
}
return value;
}
throw new python.Error('Model file does not contain ' + key + '.');
};
this.version = key_vals.get('version') || '';
this.num_class = atoi('num_class');
this.num_tree_per_iteration = atoi('num_tree_per_iteration', this.num_class);
this.label_index = atoi('label_index');
this.max_feature_idx = atoi('max_feature_idx');
if (key_vals.has('average_output')) {
this.average_output = true;
}
this.feature_names = list('feature_names', this.max_feature_idx + 1);
this.feature_infos = list('feature_infos', this.max_feature_idx + 1);
if (key_vals.has('monotone_constraints')) {
this.monotone_constraints = list('monotone_constraints', this.max_feature_idx + 1, true);
}
if (key_vals.has('objective')) {
this.objective = key_vals.get('objective');
}
let tree = null;
// let lineNumber = 0;
while (lines.length > 0) {
// lineNumber++;
const text = lines.shift();
const line = text.trim();
if (line.length === 0) {
continue;
}
if (line.startsWith('Tree=')) {
tree = { index: parseInt(line.split('=').pop(), 10) };
this.models.push(tree);
continue;
}
if (line === 'end of trees') {
break;
}
const param = line.split('=');
if (param.length !== 2) {
throw new python.Error("Invalid property '" + line + "'.");
}
const name = param[0].trim();
const value = param[1].trim();
tree[name] = value;
}
const ss = [];
let is_inparameter = false;
while (lines.length > 0) {
const text = lines.shift();
const line = text.trim();
if (line === 'parameters:') {
is_inparameter = true;
continue;
}
else if (line === 'end of parameters') {
break;
}
else if (is_inparameter) {
ss.push(line);
}
}
if (ss.length > 0) {
this.loaded_parameter = ss.join('\n');
}
}
});
this.registerType('nolearn.lasagne.base.BatchIterator', class {});
this.registerType('nolearn.lasagne.base.Layers', class {});
this.registerType('nolearn.lasagne.base.NeuralNet', class {});
this.registerType('nolearn.lasagne.base.TrainSplit', class {});
this.registerType('nolearn.lasagne.handlers.PrintLayerInfo', class {});
this.registerType('nolearn.lasagne.handlers.PrintLog', class {});
this.registerType('numpy.ndarray', class {
constructor(shape, dtype, buffer, offset, strides, order) {
this.shape = shape;
this.dtype = dtype;
this.data = buffer !== undefined ? buffer : null;
this.offset = offset !== undefined ? offset : 0;
this.strides = strides !== undefined ? strides : null;
this.order = offset !== undefined ? order : null;
this.flags = {};
}
__setstate__(state) {
this.version = state[0];
this.shape = state[1];
this.dtype = state[2];
this.flags.fnc = state[3];
this.data = state[4];
}
__read__(unpickler) {
const dims = (this.shape || []).reduce((a, b) => a * b, 1);
const size = this.dtype.itemsize * dims;
if (typeof this.data == 'string') {
this.data = unpickler.unescape(this.data, size);
if (this.data.length != size) {
throw new python.Error('Invalid string array data size.');
}
}
else {
if (this.data.length != size) {
// throw new pytorch.Error('Invalid array data size.');
}
}
return this;
}
tobytes() {
return this.data;
}
});
this.registerType('numpy.ma.core.MaskedArray', class extends numpy.ndarray {
constructor(data /*, mask, dtype, copy, subok, ndmin, fill_value, keep_mask, hard_mask, shrink, order */) {
super(data.shape, data.dtype, data.data);
}
});
this.registerType('numpy.core.memmap.memmap', class extends numpy.ndarray {
constructor(shape, dtype) {
super(shape, dtype);
}
});
this.registerType('pathlib.PosixPath', class {
constructor() {
this.path = Array.from(arguments).join('/');
}
});
this.registerType('sklearn.calibration._CalibratedClassifier', class {});
this.registerType('sklearn.calibration._SigmoidCalibration', class {});
this.registerType('sklearn.calibration.CalibratedClassifierCV', class {});
this.registerType('sklearn.compose._column_transformer.ColumnTransformer', class {});
this.registerType('sklearn.compose._target.TransformedTargetRegressor', class {});
this.registerType('sklearn.cluster._dbscan.DBSCAN', class {});
this.registerType('sklearn.cluster._kmeans.KMeans', class {});
this.registerType('sklearn.decomposition._pca.PCA', class {});
this.registerType('sklearn.decomposition.PCA', class {});
this.registerType('sklearn.decomposition.pca.PCA', class {});
this.registerType('sklearn.decomposition._truncated_svd.TruncatedSVD', class {});
this.registerType('sklearn.decomposition.truncated_svd.TruncatedSVD', class {});
this.registerType('sklearn.discriminant_analysis.LinearDiscriminantAnalysis', class {});
this.registerType('sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis', class {});
this.registerType('sklearn.dummy.DummyClassifier', class {});
this.registerType('sklearn.dummy.DummyRegressor', class {});
this.registerType('sklearn.externals.joblib.numpy_pickle.NumpyArrayWrapper', class {
constructor(/* subtype, shape, dtype */) {
}
__setstate__(state) {
this.subclass = state.subclass;
this.dtype = state.dtype;
this.shape = state.shape;
this.order = state.order;
this.allow_mmap = state.allow_mmap;
}
__read__(unpickler) {
if (this.dtype.name == 'object') {
return unpickler.load((name, args) => self.invoke(name, args), null);
}
else {
const size = this.dtype.itemsize * this.shape.reduce((a, b) => a * b, 1);
this.data = unpickler.read(size);
}
return self.invoke(this.subclass, [ this.shape, this.dtype, this.data ]);
}
});
this.registerType('sklearn.externals.joblib.numpy_pickle.NDArrayWrapper', class {
constructor(/* subtype, shape, dtype */) {
}
__setstate__(state) {
this.subclass = state.subclass;
this.filename = state.state;
this.allow_mmap = state.allow_mmap;
}
__read__(/* unpickler */) {
return this; // return self.invoke(this.subclass, [ this.shape, this.dtype, this.data ]);
}
});
this.registerType('sklearn.ensemble._bagging.BaggingClassifier', class {});
this.registerType('sklearn.ensemble._forest.RandomForestRegressor', class {});
this.registerType('sklearn.ensemble._forest.RandomForestClassifier', class {});
this.registerType('sklearn.ensemble._forest.ExtraTreesClassifier', class {});
this.registerType('sklearn.ensemble._gb_losses.BinomialDeviance', class {});
this.registerType('sklearn.ensemble._gb_losses.LeastSquaresError', class {});
this.registerType('sklearn.ensemble._gb_losses.MultinomialDeviance', class {});
this.registerType('sklearn.ensemble._gb.GradientBoostingClassifier', class {});
this.registerType('sklearn.ensemble._gb.GradientBoostingRegressor', class {});
this.registerType('sklearn.ensemble._iforest.IsolationForest', class {});
this.registerType('sklearn.ensemble._stacking.StackingClassifier', class {});
this.registerType('sklearn.ensemble._voting.VotingClassifier', class {});
this.registerType('sklearn.ensemble.forest.RandomForestClassifier', class {});
this.registerType('sklearn.ensemble.forest.RandomForestRegressor', class {});
this.registerType('sklearn.ensemble.forest.ExtraTreesClassifier', class {});
this.registerType('sklearn.ensemble.gradient_boosting.BinomialDeviance', class {});
this.registerType('sklearn.ensemble.gradient_boosting.GradientBoostingClassifier', class {});
this.registerType('sklearn.ensemble.gradient_boosting.LogOddsEstimator', class {});
this.registerType('sklearn.ensemble.gradient_boosting.MultinomialDeviance', class {});
this.registerType('sklearn.ensemble.gradient_boosting.PriorProbabilityEstimator', class {});
this.registerType('sklearn.ensemble.weight_boosting.AdaBoostClassifier', class {});
this.registerType('sklearn.feature_extraction._hashing.FeatureHasher', class {});
this.registerType('sklearn.feature_extraction.text.CountVectorizer', class {});
this.registerType('sklearn.feature_extraction.text.HashingVectorizer', class {});
this.registerType('sklearn.feature_extraction.text.TfidfTransformer', class {});
this.registerType('sklearn.feature_extraction.text.TfidfVectorizer', class {});
this.registerType('sklearn.feature_selection._from_model.SelectFromModel', class {});
this.registerType('sklearn.feature_selection._univariate_selection.SelectKBest', class {});
this.registerType('sklearn.feature_selection._univariate_selection.SelectPercentile', class {});
this.registerType('sklearn.feature_selection._variance_threshold.VarianceThreshold', class {});
this.registerType('sklearn.feature_selection.univariate_selection.SelectKBest', class {});
this.registerType('sklearn.feature_selection.variance_threshold.VarianceThreshold', class {});
this.registerType('sklearn.gaussian_process.gpc.GaussianProcessClassifier', class {});
this.registerType('sklearn.gaussian_process.kernels.ConstantKernel', class {});
this.registerType('sklearn.gaussian_process.kernels.Product', class {});
this.registerType('sklearn.gaussian_process.kernels.RBF', class {});
this.registerType('sklearn.grid_search._CVScoreTuple', class {});
this.registerType('sklearn.grid_search.GridSearchCV', class {});
this.registerType('sklearn.impute._base.SimpleImputer', class {});
this.registerType('sklearn.impute.SimpleImputer', class {});
this.registerType('sklearn.isotonic.IsotonicRegression', class {});
this.registerType('sklearn.linear_model._base.LinearRegression', class {});
this.registerType('sklearn.linear_model._bayes.BayesianRidge', class {});
this.registerType('sklearn.linear_model._coordinate_descent.ElasticNetCV', class {});
this.registerType('sklearn.linear_model._coordinate_descent.ElasticNet', class {});
this.registerType('sklearn.linear_model._logistic.LogisticRegression', class {});
this.registerType('sklearn.linear_model._ridge.Ridge', class {});
this.registerType('sklearn.linear_model._sgd_fast.Hinge', class {});
this.registerType('sklearn.linear_model._sgd_fast.Log', class {});
this.registerType('sklearn.linear_model._sgd_fast.ModifiedHuber', class {});
this.registerType('sklearn.linear_model._sgd_fast.SquaredHinge', class {});
this.registerType('sklearn.linear_model._stochastic_gradient.SGDClassifier', class {});
this.registerType('sklearn.linear_model.base.LinearRegression', class {});
this.registerType('sklearn.linear_model.sgd_fast.Hinge', class {});
this.registerType('sklearn.linear_model.LogisticRegression', class {});
this.registerType('sklearn.linear_model.logistic.LogisticRegression', class {});
this.registerType('sklearn.linear_model.logistic.LogisticRegressionCV', class {});
this.registerType('sklearn.linear_model.LassoLars​', class {});
this.registerType('sklearn.linear_model.ridge.Ridge', class {});
this.registerType('sklearn.linear_model.sgd_fast.Log', class {});
this.registerType('sklearn.linear_model.stochastic_gradient.SGDClassifier', class {});
this.registerType('sklearn.metrics._scorer._PredictScorer', class {});
this.registerType('sklearn.metrics.scorer._PredictScorer', class {});
this.registerType('sklearn.metrics._scorer._ThresholdScorer', class {});
this.registerType('sklearn.mixture._bayesian_mixture.BayesianGaussianMixture', class {});
this.registerType('sklearn.model_selection._search.GridSearchCV', class {});
this.registerType('sklearn.model_selection._search.RandomizedSearchCV', class {});
this.registerType('sklearn.model_selection._split.KFold', class {});
this.registerType('sklearn.model_selection._split.StratifiedKFold', class {});
this.registerType('sklearn.multiclass.OneVsRestClassifier', class {});
this.registerType('sklearn.multioutput.MultiOutputClassifier', class {});
this.registerType('sklearn.multioutput.MultiOutputRegressor', class {});
this.registerType('sklearn.naive_bayes.BernoulliNB', class {});
this.registerType('sklearn.naive_bayes.ComplementNB', class {});
this.registerType('sklearn.naive_bayes.GaussianNB', class {});
this.registerType('sklearn.naive_bayes.MultinomialNB', class {});
this.registerType('sklearn.neighbors._classification.KNeighborsClassifier', class {});
this.registerType('sklearn.neighbors._dist_metrics.newObj', class {});
this.registerType('sklearn.neighbors._kd_tree.newObj', class {});
this.registerType('sklearn.neighbors._regression.KNeighborsRegressor', class {});
this.registerType('sklearn.neighbors.classification.KNeighborsClassifier', class {});
this.registerType('sklearn.neighbors.dist_metrics.newObj', class {});
this.registerType('sklearn.neighbors.kd_tree.newObj', class {});
this.registerType('sklearn.neighbors.KNeighborsClassifier', class {});
this.registerType('sklearn.neighbors.KNeighborsRegressor', class {});
this.registerType('sklearn.neighbors.regression.KNeighborsRegressor', class {});
this.registerType('sklearn.neighbors.unsupervised.NearestNeighbors', class {});
this.registerType('sklearn.neural_network._multilayer_perceptron.MLPClassifier', class {});
this.registerType('sklearn.neural_network._multilayer_perceptron.MLPRegressor', class {});
this.registerType('sklearn.neural_network._stochastic_optimizers.AdamOptimizer', class {});
this.registerType('sklearn.neural_network._stochastic_optimizers.SGDOptimizer', class {});
this.registerType('sklearn.neural_network.rbm.BernoulliRBM', class {});
this.registerType('sklearn.neural_network.multilayer_perceptron.MLPClassifier', class {});
this.registerType('sklearn.neural_network.multilayer_perceptron.MLPRegressor', class {});
this.registerType('sklearn.neural_network.stochastic_gradient.SGDClassifier', class {});
this.registerType('sklearn.pipeline.Pipeline', class {});
this.registerType('sklearn.pipeline.FeatureUnion', class {});
this.registerType('sklearn.preprocessing._data.MinMaxScaler', class {});
this.registerType('sklearn.preprocessing._data.MaxAbsScaler', class {});
this.registerType('sklearn.preprocessing._data.Normalizer', class {});
this.registerType('sklearn.preprocessing._data.PolynomialFeatures', class {});
this.registerType('sklearn.preprocessing._data.QuantileTransformer', class {});
this.registerType('sklearn.preprocessing._data.RobustScaler', class {});
this.registerType('sklearn.preprocessing._data.StandardScaler', class {});
this.registerType('sklearn.preprocessing._discretization.KBinsDiscretizer', class {});
this.registerType('sklearn.preprocessing._encoders.OneHotEncoder', class {});
this.registerType('sklearn.preprocessing._function_transformer.FunctionTransformer', class {});
this.registerType('sklearn.preprocessing._label.LabelBinarizer', class {});
this.registerType('sklearn.preprocessing._label.LabelEncoder', class {});
this.registerType('sklearn.preprocessing.data.Binarizer', class {});
this.registerType('sklearn.preprocessing.data.MaxAbsScaler', class {});
this.registerType('sklearn.preprocessing.data.MinMaxScaler', class {});
this.registerType('sklearn.preprocessing.data.Normalizer', class {});
this.registerType('sklearn.preprocessing.data.OneHotEncoder', class {});
this.registerType('sklearn.preprocessing.data.PolynomialFeatures', class {});
this.registerType('sklearn.preprocessing.data.PowerTransformer', class {});
this.registerType('sklearn.preprocessing.data.RobustScaler', class {});
this.registerType('sklearn.preprocessing.data.QuantileTransformer', class {});
this.registerType('sklearn.preprocessing.data.StandardScaler', class {});
this.registerType('sklearn.preprocessing.imputation.Imputer', class {});
this.registerType('sklearn.preprocessing.label.LabelBinarizer', class {});
this.registerType('sklearn.preprocessing.label.LabelEncoder', class {});
this.registerType('sklearn.preprocessing.label.MultiLabelBinarizer', class {});
this.registerType('sklearn.svm._classes.LinearSVC', class {});
this.registerType('sklearn.svm._classes.SVC', class {});
this.registerType('sklearn.svm._classes.SVR', class {});
this.registerType('sklearn.svm.classes.LinearSVC', class {});
this.registerType('sklearn.svm.classes.OneClassSVM', class {});
this.registerType('sklearn.svm.classes.SVC', class {});
this.registerType('sklearn.svm.classes.SVR', class {});
this.registerType('sklearn.tree._classes.DecisionTreeClassifier', class {});
this.registerType('sklearn.tree._classes.DecisionTreeRegressor', class {});
this.registerType('sklearn.tree._classes.ExtraTreeClassifier', class {});
this.registerType('sklearn.tree._classes.ExtraTreeRegressor', class {});
this.registerType('sklearn.tree._tree.Tree', class {
constructor(n_features, n_classes, n_outputs) {
this.n_features = n_features;
this.n_classes = n_classes;
this.n_outputs = n_outputs;
}
__setstate__(state) {
this.max_depth = state.max_depth;
this.node_count = state.node_count;
this.nodes = state.nodes;
this.values = state.values;
}
});
this.registerType('sklearn.tree.tree.DecisionTreeClassifier', class {});
this.registerType('sklearn.tree.tree.DecisionTreeRegressor', class {});
this.registerType('sklearn.tree.tree.ExtraTreeClassifier', class {});
this.registerType('sklearn.utils.Bunch', class {});
this.registerType('sklearn.utils.deprecation.DeprecationDict', class {});
this.registerType('re.Pattern', function(pattern, flags) {
this.pattern = pattern;
this.flags = flags;
});
this.registerType('spacy._ml.PrecomputableAffine', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('spacy.syntax._parser_model.ParserModel', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.describe.Biases', class {
__setstate__(state) {
Object.assign(this, state);
}
});
this.registerType('thinc.describe.Dimension', class {
__setstate__(state) {
Object.assign(this, state);
}
});
this.registerType('thinc.describe.Gradient', class {
__setstate__(state) {
Object.assign(this, state);
}
});
this.registerType('thinc.describe.Weights', class {
__setstate__(state) {
Object.assign(this, state);
}
});
this.registerType('thinc.describe.Synapses', class {
__setstate__(state) {
Object.assign(this, state);
}
});
this.registerType('thinc.neural._classes.affine.Affine', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.neural._classes.convolution.ExtractWindow', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.neural._classes.feature_extracter.FeatureExtracter', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.neural._classes.feed_forward.FeedForward', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.neural._classes.function_layer.FunctionLayer', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.neural._classes.hash_embed.HashEmbed', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.neural._classes.layernorm.LayerNorm', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.neural._classes.maxout.Maxout', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.neural._classes.resnet.Residual', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.neural._classes.softmax.Softmax', class {
__setstate__(state) {
Object.assign(this, python.Unpickler.open(state).load((name, args) => self.invoke(name, args), null));
}
});
this.registerType('thinc.neural.mem.Memory', class {
});
this.registerType('thinc.neural.ops.NumpyOps', class {
});
this.registerType('types.CodeType', class {
constructor(/* args */) {
}
});
this.registerType('types.MethodType', class {
constructor(/* args */) {
}
});
this.registerType('types.ObjectType', builtins.object);
this.registerType('xgboost.compat.XGBoostLabelEncoder', class {});
this.registerType('xgboost.core.Booster', class {});
this.registerType('xgboost.sklearn.XGBClassifier', class {});
this.registerType('xgboost.sklearn.XGBRegressor', class {});
this.registerFunction('__builtin__.bytearray', function(source, encoding /*, errors */) {
if (source) {
if (encoding === 'latin-1') {
const array = new Uint8Array(source.length);
for (let i = 0; i < source.length; i++) {
array[i] = source.charCodeAt(i);
}
return array;
}
throw new python.Error("Unsupported bytearray encoding '" + JSON.stringify(encoding) + "'.");
}
return [];
});
this.registerFunction('__builtin__.bytes', function(source, encoding /*, errors */) {
if (source) {
if (encoding === 'latin-1') {
const array = new Uint8Array(source.length);
for (let i = 0; i < source.length; i++) {
array[i] = source.charCodeAt(i);
}
return array;
}
throw new python.Error("Unsupported bytearray encoding '" + JSON.stringify(encoding) + "'.");
}
return [];
});
this.registerFunction('__builtin__.set', function(iterable) {
return iterable ? iterable : [];
});
this.registerFunction('__builtin__.frozenset', function(iterable) {
return iterable ? iterable : [];
});
this.registerFunction('__builtin__.getattr', function(obj, name, defaultValue) {
if (Object.prototype.hasOwnProperty.call(obj, name)) {
return obj[name];
}
return defaultValue;
});
this.registerFunction('__builtin__.slice', function(start, stop , step) {
return [ start, stop, step ];
});
this.registerFunction('__builtin__.type', function(obj) {
return obj ? obj.__class__ : undefined;
});
this.registerFunction('_codecs.encode', function(obj /*, econding */) {
return obj;
});
this.registerFunction('builtins.bytearray', function(data) {
return { data: data };
});
this.registerFunction('builtins.getattr', function(obj, name, defaultValue) {
if (Object.prototype.hasOwnProperty.call(obj, name)) {
return obj[name];
}
return defaultValue;
});
this.registerFunction('builtins.set', function(iterable) {
return iterable ? iterable : [];
});
this.registerFunction('builtins.slice', function(start, stop, step) {
return { start: start, stop: stop, step: step };
});
this.registerFunction('cloudpickle.cloudpickle._builtin_type', function(name) {
return name;
});
this.registerFunction('collections.Counter', function(/* iterable */) {
return { __module__: 'collections', __name__: 'Counter' };
});
this.registerFunction('collections.defaultdict', function(/* default_factory */) {
return {};
});
this.registerFunction('copy_reg._reconstructor', function(cls, base, state) {
// copyreg._reconstructor in Python 3
if (base === '__builtin__.object' || base === builtins.object) {
return self.invoke(cls, []);
}
else if (base === '__builtin__.tuple' || base === builtins.tuple) {
const obj = self.invoke(cls, []);
for (let i = 0; i < state.length; i++) {
obj[i] = state[i];
}
return obj;
}
throw new python.Error("Unknown copy_reg._reconstructor base type '" + base + "'.");
});
this.registerFunction('dill._dill._create_cell', function(/* args */) {
return function() {
// TODO
};
});
this.registerFunction('dill._dill._create_code', function(args) {
return self.invoke('types.CodeType', [ args ]);
});
this.registerFunction('dill._dill._create_function', function(/* fcode, fglobals, fname, fdefaults, fclosure, fdict, fkwdefaults */) {
return function() {
// TODO
};
});
this.registerFunction('dill._dill._get_attr', function(self, name) {
if (Object.prototype.hasOwnProperty.call(self, name)) {
return self[name];
}
return undefined;
});
this.registerFunction('dill._dill._import_module', function(import_name, safe) {
try {
return self.context.getx(import_name);
}
catch (err) {
if (safe) {
return null;
}
}
});
this.registerFunction('dill.dill._load_type', function(name) {
return self.context.getx('types.' + name);
});
this.registerFunction('dill._dill._load_type', function(name) {
return self.context.getx('types.' + name);
});
this.registerFunction('getattr', function(obj, name, defaultValue) {
if (Object.prototype.hasOwnProperty.call(obj, name)) {
return obj[name];
}
return defaultValue;
});
this.registerFunction('numpy.core._multiarray_umath._reconstruct', function(subtype, shape, dtype) {
return self.invoke(subtype, [ shape, dtype ]);
});
this.registerFunction('numpy.core.multiarray._reconstruct', function(subtype, shape, dtype) {
return self.invoke(subtype, [ shape, dtype ]);
});
this.registerFunction('numpy.core.multiarray.scalar', function(dtype, rawData) {
let data = rawData;
if (typeof rawData === 'string' || rawData instanceof String) {
data = new Uint8Array(rawData.length);
for (let i = 0; i < rawData.length; i++) {
data[i] = rawData.charCodeAt(i);
}
}
const dataView = new DataView(data.buffer, data.byteOffset, data.byteLength);
switch (dtype.name) {
case 'float32':
return dataView.getFloat32(0, true);
case 'float64':
return dataView.getFloat64(0, true);
case 'uint8':
return dataView.getUint8(0, true);
case 'int8':
return dataView.getInt8(0, true);
case 'int16':
return dataView.getInt16(0, true);
case 'int32':
return dataView.getInt32(0, true);
case 'int64':
return dataView.getInt64(0, true);
case 'bool':
return dataView.getInt8(0, true) ? true : false;
}
throw new python.Error("Unknown scalar type '" + dtype.name + "'.");
});
this.registerFunction('numpy.core._multiarray_umath.scalar', function(dtype, rawData) {
let data = rawData;
if (typeof rawData === 'string') {
data = new Uint8Array(rawData.length);
for (let i = 0; i < rawData.length; i++) {
data[i] = rawData.charCodeAt(i);
}
}
const dataView = new DataView(data.buffer, data.byteOffset, data.byteLength);
switch (dtype.name) {
case 'uint8':
return dataView.getUint8(0);
case 'float32':
return dataView.getFloat32(0, true);
case 'float64':
return dataView.getFloat64(0, true);
case 'int8':
return dataView.getInt8(0, true);
case 'int16':
return dataView.getInt16(0, true);
case 'int32':
return dataView.getInt32(0, true);
case 'int64':
return dataView.getInt64(0, true);
}
throw new python.Error("Unknown scalar type '" + dtype.name + "'.");
});
this.registerFunction('numpy.load', function(file) {
// https://github.com/numpy/numpy/blob/main/numpy/lib/format.py
const signature = [ 0x93, 0x4E, 0x55, 0x4D, 0x50, 0x59 ];
if (!file.read(6).every((v, i) => v == signature[i])) {
throw new numpy.Error('Invalid signature.');
}
const major = file.read(1)[0];
const minor = file.read(1)[0];
if (major > 3) {
throw new python.Error("Invalid version '" + [ major, minor ].join('.') + "'.");
}
const buffer = new Uint8Array([ 0, 0, 0, 0 ]);
buffer.set(file.read(major >= 2 ? 4 : 2), 0);
const header_length = buffer[3] << 24 | buffer[2] << 16 | buffer[1] << 8 | buffer[0];
let header = file.read(header_length);
const decoder = new TextDecoder(major >= 3 ? 'utf-8' : 'ascii');
header = decoder.decode(header);
header = JSON.parse(header.replace(/\(/,'[').replace(/\)/,']').replace('[,','[1,]').replace(',]',',1]').replace(/'/g, '"').replace(/:\s*False\s*,/,':false,').replace(/:\s*True\s*,/,':true,').replace(/,\s*\}/, ' }'));
if (!header.descr || header.descr.length < 2) {
throw new numpy.Error("Missing property 'descr'.");
}
if (!header.shape) {
throw new numpy.Error("Missing property 'shape'.");
}
const shape = header.shape;
const dtype = self.invoke('numpy.dtype', [ header.descr.substring(1) ]);
dtype.byteorder = header.descr[0];
let data = null;
switch (dtype.byteorder) {
case '|': {
data = file.read();
break;
}
case '>':
case '<': {
if (header.descr.length !== 3) {
throw new numpy.Error("Unsupported data type '" + header.descr + "'.");
}
const count = shape.length === 0 ? 1 : shape.reduce((a, b) => a * b, 1);
data = file.read(dtype.itemsize * count);
break;
}
default: {
throw new numpy.Error("Unsupported data type '" + header.descr + "'.");
}
}
if (header.fortran_order) {
data = null;
}
return self.invoke('numpy.ndarray', [ shape, dtype, data ]);
});
this.registerFunction('numpy.save', function(file, arr) {
const descr = arr.dtype.str;
if (descr[0] !== '<' && descr[0] !== '>') {
throw new numpy.Error("Unknown byte order '" + descr + "'.");
}
if (descr.length !== 3 || (descr[1] !== 'f' && descr[1] !== 'i' && descr[1] !== 'u' && descr.substring(1) !== 'b1')) {
throw new numpy.Error("Unsupported data type '" + descr + "'.");
}
let shape = '';
switch (arr.shape.length) {
case 0: shape = '()'; break;
case 1: shape = '(' + arr.shape[0].toString() + ',)'; break;
default: shape = '(' + arr.shape.map((dimension) => dimension.toString()).join(', ') + ')'; break;
}
const properties = [
"'descr': '" + descr + "'",
"'fortran_order': False",
"'shape': " + shape
];
let header = '{ ' + properties.join(', ') + ' }';
header += ' '.repeat(64 - ((header.length + 2 + 8 + 1) & 0x3f)) + '\n';
const encoder = new TextEncoder('ascii');
file.write([ 0x93, 0x4E, 0x55, 0x4D, 0x50, 0x59, 0x01, 0x00 ]); // '\\x93NUMPY' + version
file.write([ header.length & 0xff, (header.length >> 8) & 0xff ]);
file.write(encoder.encode(header));
file.write(arr.tobytes());
});
this.registerFunction('numpy.asarray', function(a, dtype) {
const encode = (context, data, dim) => {
const size = context.shape[dim];
const littleendian = context.littleendian;
if (dim == context.shape.length - 1) {
for (let i = 0; i < size; i++) {
switch (context.dtype) {
case 'f2':
context.view.setFloat16(context.position, data[i], littleendian);
break;
case 'f4':
context.view.setFloat32(context.position, data[i], littleendian);
break;
case 'f8':
context.view.setFloat64(context.position, data[i], littleendian);
break;
case 'i1':
context.view.setInt8(context.position, data[i], littleendian);
break;
case 'i2':
context.view.setInt16(context.position, data[i], littleendian);
break;
case 'i4':
context.view.setInt32(context.position, data[i], littleendian);
break;
case 'i8':
context.view.setInt64(context.position, data[i], littleendian);
break;
case 'u1':
context.view.setUint8(context.position, data[i], littleendian);
break;
case 'u2':
context.view.setUint16(context.position, data[i], littleendian);
break;
case 'u4':
context.view.setUint32(context.position, data[i], littleendian);
break;
case 'u8':
context.view.setUint64(context.position, data[i], littleendian);
break;
}
context.position += context.itemsize;
}
}
else {
for (let j = 0; j < size; j++) {
encode(context, data[j], dim + 1);
}
}
};
const array_size = (value) => {
if (value.every((item) => Array.isArray(item))) {
const dims = value.map((item) => array_size(item));
const dim = dims[0];
for (let i = 1; i < dims.length; i++) {
if (dim.length === dims[i].length) {
if (!dims[i].every((value, i) => value ===dim[i])) {
throw new python.Error('Invalid array shape.');
}
}
}
return [ value.length ].concat(dim);
}
return [ value.length ];
};
const shape = Array.isArray(a) ? array_size(a) : [];
const size = dtype.itemsize * shape.reduce((a, b) => a * b, 1);
const context = {
position: 0,
itemsize: dtype.itemsize,
dtype: dtype.str.substring(1),
littleendian: dtype.str[0],
shape: shape,
data: new Uint8Array(size)
};
context.view = new DataView(context.data.buffer, context.data.byteOffset, size);
encode(context, a, 0);
return self.invoke('numpy.ndarray', [ shape, dtype, context.data ]);
});
this.registerFunction('numpy.ma.core._mareconstruct', function(subtype, baseclass, baseshape, basetype) {
const data = self.invoke(baseclass, [ baseshape, basetype ]);
// = ndarray.__new__(ndarray, baseshape, make_mask_descr(basetype))
const mask = self.invoke('numpy.ndarray', [ baseshape, '' ]);
return self.invoke(subtype, [ data, mask, basetype ]);
});
this.registerFunction('numpy.random.__RandomState_ctor', function() {
return {};
});
this.registerFunction('numpy.random._pickle.__randomstate_ctor', function() {
return {};
});
this.registerFunction('numpy.core.numeric._frombuffer', function(/* buf, dtype, shape, order */) {
return {};
});
this.registerFunction('re._compile', function(pattern, flags) {
return self.invoke('re.Pattern', [ pattern, flags ]);
});
this.registerFunction('srsly.cloudpickle.cloudpickle._builtin_type', function(name) {
return function() {
return self.invoke('types.' + name, arguments);
};
});
}
get context() {
return this._context;
}
source(file) {
return this._sources.has(file) ? this._sources.get(file) : null;
}
debug(/* file */) {
}
parse(file) {
const buffer = this.source(file);
if (buffer) {
const debug = this.debug(file);
const code = this._utf8Decoder.decode(buffer);
const reader = new python.Parser(code, file, debug);
const program = reader.parse();
if (!program) {
throw new python.Error("Module '" + file + "' parse error.");
}
return program;
}
return null;
}
package(name) {
const index = name.lastIndexOf('.');
if (index > 0) {
this.package(name.substring(0, index));
}
if (!this._packages.has(name)) {
const file = 'code/' + name.split('.').join('/') + '.py';
const program = this.parse(file);
if (program) {
let globals = this._context.getx(name);
if (globals === undefined) {
globals = {};
this._context.setx(name, globals);
}
globals.__class__ = this._context.scope.builtins.module;
globals.__name__ = name;
globals.__file__ = file;
this._packages.set(name, globals);
const context = this._context.push(globals);
this.block(program.body, context);
}
}
return this._packages.get(name);
}
type(name) {
const type = this._context.getx(name);
if (type !== undefined) {
return type;
}
const parts = name.split('.');
const className = parts.pop();
const moduleName = parts.join('.');
const module = this.package(moduleName);
if (module) {
return module[className];
}
return null;
}
invoke(name, args) {
const target = name.__class__ ? name : this.type(name);
if (target) {
if (target.__class__ === this._context.scope.builtins.type) {
if (target.prototype && target.prototype.__class__ === target) {
return Reflect.construct(target, args);
}
const obj = {};
obj.__proto__ = target;
if (obj.__init__ && typeof obj.__init__ === 'function') {
obj.__init__.apply(obj, args);
}
return obj;
}
else if (target.__class__ === this._context.scope.builtins.function) {
if (target.__call__) {
return target.__call__(args);
}
else {
return target.apply(null, args);
}
}
}
this._raiseUnkownName(name);
this.registerType(name, class {});
return this.invoke(name, []);
}
call(target, name, args, context) {
const callTarget = this._target(target, context);
const callArguments = args.map((argument) => this.expression(argument, context));
if (!callTarget || (name !== null && !callTarget[name])) {
if (name === '__new__' && callArguments.length === 1 && callArguments[0] == callTarget) {
name = null;
callArguments.shift();
}
else {
const targetName = python.Utility.target(target) + '.' + name;
if (this.type(targetName)) {
return this.invoke(targetName, callArguments);
}
throw new python.Error("Unsupported function '" + targetName + "'.");
}
}
const func = name ? callTarget[name] : callTarget;
if (func.__class__ === this._context.scope.builtins.type) {
if (func.prototype && func.prototype.__class__ === func) {
return Reflect.construct(func, args);
}
const obj = {};
obj.__proto__ = func;
obj.__class__ = func;
if (obj.__init__ && typeof obj.__init__ === 'function') {
obj.__init__.apply(obj, args);
}
return obj;
}
if (func.__class__ === this._context.scope.builtins.function) {
if (func.__call__) {
return func.__call__(callArguments);
}
}
if (func.__class__ === this._context.scope.builtins.method) {
if (func.__call__) {
return func.__call__([ callTarget ].concat(callArguments));
}
}
if (typeof func === 'function') {
return func.apply(callTarget, callArguments);
}
throw new python.Error("Unsupported call expression.");
}
apply(method, args, context) {
const locals = Array.prototype.slice.call(args);
context = context.push();
for (const parameter of method.parameters) {
let value = locals.shift();
if (value === undefined && parameter.initializer) {
value = this.expression(parameter.initializer, context);
}
context.set(parameter.name, value);
}
return this.block(method.body.statements, context);
}
block(statements, context) {
statements = Array.prototype.slice.call(statements);
while (statements.length > 0) {
const statement = statements.shift();
const value = this.statement(statement, context);
if (value !== undefined) {
return value;
}
}
}
statement(statement, context) {
switch (statement.type) {
case 'pass': {
break;
}
case 'return': {
return this.expression(statement.expression, context);
}
case 'def': {
const module = context.get('__name__');
const self = this;
const parent = context.get('__class__');
let type = null;
if (parent === this._context.scope.builtins.type) {
type = this._context.scope.builtins.method;
}
else if (parent === this._context.scope.builtins.module) {
type = this._context.scope.builtins.function;
}
else {
throw new python.Error('Invalid function scope.');
}
const func = {
__class__: type,
__globals__: context,
__module__: module,
__name__: statement.name,
__code__: statement,
__call__: function(args) {
return self.apply(this.__code__, args, this.__globals__);
}
};
context.set(statement.name, func);
break;
}
case 'class': {
const scope = {
__class__:this._context.scope.builtins.type,
__module__: context.get('__name__'),
__name__: statement.name,
};
context.set(statement.name, scope);
context = context.push(scope);
this.block(statement.body.statements, context);
context = context.pop();
break;
}
case 'var': {
context.set(statement.name, statement.initializer ? this.expression(statement.initializer, context) : undefined);
break;
}
case '=': {
this.expression(statement, context);
break;
}
case 'if': {
const condition = this.expression(statement.condition, context);
if (condition === true || condition) {
const value = this.block(statement.then.statements, context);
if (value !== undefined) {
return value;
}
break;
}
else if (condition === false) {
const value = this.block(statement.else.statements, context);
if (value !== undefined) {
return value;
}
break;
}
throw new python.Error("Unknown condition.");
}
case 'for': {
if (statement.target.length == 1 &&
statement.variable.length === 1 && statement.variable[0].type === 'id') {
const range = this.expression(statement.target[0], context);
const variable = statement.variable[0];
for (const current of range) {
this.statement({ type: '=', target: variable, expression: { type: 'number', value: current }}, context);
const value = this.block(statement.body.statements, context);
if (value !== undefined) {
return value;
}
}
break;
}
throw new python.Error("Unsupported 'for' statement.");
}
case 'while': {
const condition = this.expression(statement.condition, context);
if (condition) {
const value = this.block(statement.body.statements, context);
if (value !== undefined) {
return value;
}
}
break;
}
case 'call': {
this.expression(statement, context);
break;
}
case 'import': {
for (const module of statement.modules) {
const moduleName = python.Utility.target(module.name);
const globals = this.package(moduleName);
if (module.as) {
context.set(module.as, globals);
}
}
break;
}
default: {
throw new python.Error("Unknown statement '" + statement.type + "'.");
}
}
}
expression(expression, context) {
const self = context.getx('self');
switch (expression.type) {
case '=': {
const target = expression.target;
if (target.type === 'id') {
context.set(target.value, this.expression(expression.expression, context));
return;
}
else if (target.type === '[]') {
if (target.target.type === 'id' &&
target.arguments.type === 'list' &&
target.arguments.value.length === 1) {
const index = this.expression(target.arguments.value[0], context);
if (target.target.value === '__annotations__') {
context.set(target.target.value, context.get(target.target.value) || {});
}
context.get(target.target.value)[index] = this.expression(expression.expression, context);
return;
}
}
else if (target.type === '.' &&
target.member.type === 'id') {
this.expression(target.target, context)[target.member.value] = this.expression(expression.expression, context);
return;
}
else if (target.type === 'tuple') {
context.target.push(target.value);
const value = this.expression(expression.expression, context);
context.target.pop();
if (target.value.every((item) => item.type === 'id')) {
if (target.value.length < value.length) {
throw new python.Error('ValueError: too many values to unpack (expected ' + target.value.length + ', actual ' + value.length + ').');
}
if (target.value.length > value.length) {
throw new python.Error('ValueError: not enough values to unpack (expected ' + target.value.length + ', actual ' + value.length + ').');
}
for (let i = 0; i < value.length; i++) {
context.set(target.value[i].value, value[i]);
}
return;
}
}
break;
}
case 'list': {
return expression.value.map((item) => this.expression(item, context));
}
case 'string': {
return expression.value.substring(1, expression.value.length - 1);
}
case 'number': {
return Number(expression.value);
}
case '[]': {
if (expression.target.type === 'id' &&
expression.arguments.type === 'list' &&
expression.arguments.value.length === 1) {
if (context.get(expression.target.value)) {
const index = this.expression(expression.arguments.value[0], context);
const target = context.get(expression.target.value);
return target[index < 0 ? target.length + index : index];
}
}
const target = this.expression(expression.target, context);
if (target && expression.arguments.type === 'list' &&
(target.__class__ === this.context.scope.typing._TupleType ||
target.__class__ === this.context.scope.typing._SpecialGenericAlias ||
target.__class__ === this.context.scope.typing._SpecialForm)) {
const type = Object.assign({}, target);
type.__args__ = expression.arguments.value.map((arg) => this.expression(arg, context));
return type;
}
if (expression.arguments.type === 'list' && expression.arguments.value.length === 1) {
const index = this.expression(expression.arguments.value[0], context);
return target[index < 0 ? target.length + index : index];
}
break;
}
case '.': {
if (expression.member.type == 'id') {
const target = this._target(expression.target, context);
return target[expression.member.value];
}
throw new python.Error("Unsupported field expression.");
}
case 'call': {
if (expression.target.type === 'id' && expression.target.value === 'unchecked_cast' && expression.arguments.length === 2) {
return this.expression(expression.arguments[1], context);
}
if (expression.target.type === '.') {
return this.call(expression.target.target, expression.target.member.value, expression.arguments, context);
}
return this.call(expression.target, null, expression.arguments, context);
}
case 'id': {
switch (expression.value) {
case 'self': return self;
case 'None': return null;
case 'True': return true;
case 'False': return false;
}
const type = (value) => {
return value &&
(value.__class__ === this._context.scope.builtins.type ||
value.__class__ === this._context.scope.typing._TupleType ||
value.__class__ === this._context.scope.typing._SpecialGenericAlias ||
value.__class__ === this._context.scope.typing._SpecialForm);
};
const builtin = this._context.scope.builtins[expression.value];
if (type(builtin)) {
return builtin;
}
const value = context.get(expression.value);
if (value === undefined) {
const typing = this._context.scope.typing[expression.value];
if (type(typing)) {
return typing;
}
const torch = this._context.scope.torch[expression.value];
if (type(torch)) {
return torch;
}
}
return value;
}
case 'tuple': {
return expression.value.map((expression) => this.expression(expression, context));
}
case 'dict': {
const dict = {};
for (const pair of expression.value) {
if (pair.type !== 'pair') {
throw new python.Error("Unsupported dict item type '" + pair.type + "'.");
}
const key = this.expression(pair.key, context);
const value = this.expression(pair.value, context);
dict[key] = value;
}
return dict;
}
}
throw new python.Error("Unknown expression '" + expression.type + "'.");
}
_target(expression, context) {
let current = expression;
let packageName = '';
for (;;) {
if (current.type === '.' && current.member && current.member.type === 'id') {
packageName = '.' + current.member.value + packageName;
current = current.target;
}
else if (current.type === 'id' && current.value !== 'self' && current.value !== 'CONSTANTS') {
packageName = current.value + packageName;
break;
}
else {
packageName = null;
break;
}
}
if (packageName) {
let target = context.getx(packageName);
if (!target) {
target = this.package(packageName);
if (!target) {
target = context.getx(packageName);
if (!target) {
throw new python.Error("Failed to resolve module '" + packageName + "'.");
}
}
}
return target;
}
return this.expression(expression, context);
}
registerFunction(name, callback) {
if (this._context.getx(name)) {
throw new python.Error("Function '" + name + "' is already registered.");
}
const parts = name.split('.');
callback.__class__ = this._context.scope.builtins.function;
callback.__name__ = parts.pop();
callback.__module__ = parts.join('.');
this._context.setx(name, callback);
}
registerType(name, type) {
if (this._context.getx(name)) {
throw new python.Error("Class '" + name + "' is already registered.");
}
const parts = name.split('.');
type.__class__ = this._context.scope.builtins.type;
type.__name__ = parts.pop();
type.__module__ = parts.join('.');
type.prototype.__class__ = type;
this._context.setx(name, type);
}
registerModule(name) {
let scope = this._context.scope;
const items = name.split('.');
while (items.length > 0) {
const item = items.shift();
scope[item] = { __name__: name, __class__: this._context.scope.builtins.module };
scope = scope[item];
}
}
_raiseUnkownName(name) {
if (name && !this._unknownNameMap.has(name)) {
this._unknownNameMap.add(name);
const module = name.split('.').shift();
if (this._context.scope[module] && this._context.scope[module].__class__ == this._context.scope.builtins.module) {
this._exceptionCallback(new python.Error("Unknown function '" + name + "'."), false);
}
}
}
};
python.Execution.Context = class {
constructor(parent, scope) {
this._parent = parent || null;
this._scope = scope || {};
}
push(scope) {
return new python.Execution.Context(this, scope);
}
pop() {
return this._parent;
}
get scope() {
return this._scope;
}
set(name, value) {
this._scope[name] = value;
}
get(name) {
if (name in this._scope) {
return this._scope[name];
}
if (this._parent) {
return this._parent.get(name);
}
return undefined;
}
setx(name, value) {
if (typeof name !== 'string' || !name.split) {
throw new python.Error("Invalid name '" + JSON.stringify(name) + "'.");
}
const parts = name.split('.');
if (parts.length == 1) {
this.set(parts[0], value);
}
else {
let parent = this.get(parts[0]);
if (!parent) {
parent = {};
this.set(parts[0], parent);
}
parts.shift();
while (parts.length > 1) {
const part = parts.shift();
parent[part] = parent[part] || {};
parent = parent[part];
}
parent[parts[0]] = value;
}
}
getx(name) {
const parts = name.split('.');
let value = this.get(parts[0]);
if (value !== undefined) {
parts.shift();
while (parts.length > 0 && value[parts[0]]) {
value = value[parts[0]];
parts.shift();
}
if (parts.length === 0) {
return value;
}
}
return undefined;
}
get target() {
this._target = this._target || [];
return this._target;
}
};
python.Utility = class {
static target(expression) {
if (expression.type == 'id') {
return expression.value;
}
if (expression.type == '.') {
return python.Utility.target(expression.target) + '.' + python.Utility.target(expression.member);
}
return null;
}
};
python.Unpickler = class {
static open(data) {
const reader = data instanceof Uint8Array ? new python.Unpickler.BinaryReader(data) : new python.Unpickler.StreamReader(data);
if (reader.length > 2) {
const head = reader.peek(2);
if (head[0] === 0x80 && head[1] < 7) {
return new python.Unpickler(reader);
}
reader.seek(-1);
const tail = reader.peek(1);
reader.seek(0);
if (tail[0] === 0x2e) {
return new python.Unpickler(reader);
}
}
return null;
}
constructor(reader) {
this._reader = reader;
}
load(function_call, persistent_load) {
const reader = this._reader;
const marker = [];
let stack = [];
const memo = new Map();
const OpCode = python.Unpickler.OpCode;
while (reader.position < reader.length) {
const opcode = reader.byte();
switch (opcode) {
case OpCode.PROTO: {
const version = reader.byte();
if (version > 5) {
throw new python.Error("Unsupported protocol version '" + version + "'.");
}
break;
}
case OpCode.GLOBAL:
stack.push([ reader.line(), reader.line() ].join('.'));
break;
case OpCode.STACK_GLOBAL:
stack.push([ stack.pop(), stack.pop() ].reverse().join('.'));
break;
case OpCode.PUT: {
const index = parseInt(reader.line(), 10);
memo.set(index, stack[stack.length - 1]);
break;
}
case OpCode.OBJ: {
const items = stack;
stack = marker.pop();
stack.push(function_call(items.pop(), items));
break;
}
case OpCode.GET: {
const index = parseInt(reader.line(), 10);
stack.push(memo.get(index));
break;
}
case OpCode.POP:
stack.pop();
break;
case OpCode.POP_MARK:
stack = marker.pop();
break;
case OpCode.DUP:
stack.push(stack[stack.length-1]);
break;
case OpCode.PERSID:
stack.push(persistent_load(reader.line()));
break;
case OpCode.BINPERSID:
stack.push(persistent_load(stack.pop()));
break;
case OpCode.REDUCE: {
const items = stack.pop();
const type = stack.pop();
stack.push(function_call(type, items));
break;
}
case OpCode.NEWOBJ: {
const items = stack.pop();
const type = stack.pop();
stack.push(function_call(type, items));
break;
}
case OpCode.BINGET:
stack.push(memo.get(reader.byte()));
break;
case OpCode.INST: {
const module = reader.line();
const name = reader.line();
const type = module + '.' + name;
const items = stack;
stack = marker.pop();
stack.push(function_call(type, items));
break;
}
case OpCode.LONG_BINGET:
stack.push(memo.get(reader.uint32()));
break;
case OpCode.BINPUT:
memo.set(reader.byte(), stack[stack.length - 1]);
break;
case OpCode.LONG_BINPUT:
memo.set(reader.uint32(), stack[stack.length - 1]);
break;
case OpCode.BININT:
stack.push(reader.int32());
break;
case OpCode.BININT1:
stack.push(reader.byte());
break;
case OpCode.LONG:
stack.push(parseInt(reader.line(), 10));
break;
case OpCode.BININT2:
stack.push(reader.uint16());
break;
case OpCode.BINBYTES:
stack.push(reader.read(reader.int32()));
break;
case OpCode.BINBYTES8:
stack.push(reader.read(reader.int64()));
break;
case OpCode.SHORT_BINBYTES:
stack.push(reader.read(reader.byte()));
break;
case OpCode.FLOAT:
stack.push(parseFloat(reader.line()));
break;
case OpCode.BINFLOAT:
stack.push(reader.float64());
break;
case OpCode.INT: {
const value = reader.line();
if (value == '01') {
stack.push(true);
}
else if (value == '00') {
stack.push(false);
}
else {
stack.push(parseInt(value, 10));
}
break;
}
case OpCode.EMPTY_LIST:
stack.push([]);
break;
case OpCode.EMPTY_TUPLE:
stack.push([]);
break;
case OpCode.EMPTY_SET:
stack.push([]);
break;
case OpCode.ADDITEMS: {
const items = stack;
stack = marker.pop();
const obj = stack[stack.length - 1];
for (let i = 0; i < items.length; i++) {
obj.push(items[i]);
}
break;
}
case OpCode.FROZENSET: {
const items = stack;
stack = marker.pop();
stack.push(items);
break;
}
case OpCode.DICT: {
const items = stack;
stack = marker.pop();
const dict = {};
for (let i = 0; i < items.length; i += 2) {
dict[items[i]] = items[i + 1];
}
stack.push(dict);
break;
}
case OpCode.LIST: {
const items = stack;
stack = marker.pop();
stack.push(items);
break;
}
case OpCode.TUPLE: {
const items = stack;
stack = marker.pop();
stack.push(items);
break;
}
case OpCode.SETITEM: {
const value = stack.pop();
const key = stack.pop();
const obj = stack[stack.length - 1];
if (obj.__setitem__) {
obj.__setitem__(key, value);
}
else {
obj[key] = value;
}
break;
}
case OpCode.SETITEMS: {
const items = stack;
stack = marker.pop();
const obj = stack[stack.length - 1];
for (let i = 0; i < items.length; i += 2) {
if (obj.__setitem__) {
obj.__setitem__(items[i], items[i + 1]);
}
else {
obj[items[i]] = items[i + 1];
}
}
break;
}
case OpCode.EMPTY_DICT:
stack.push({});
break;
case OpCode.APPEND: {
const append = stack.pop();
stack[stack.length-1].push(append);
break;
}
case OpCode.APPENDS: {
const appends = stack;
stack = marker.pop();
const list = stack[stack.length - 1];
list.push.apply(list, appends);
break;
}
case OpCode.STRING: {
const str = reader.line();
stack.push(str.substr(1, str.length - 2));
break;
}
case OpCode.BINSTRING:
stack.push(reader.string(reader.uint32()));
break;
case OpCode.SHORT_BINSTRING:
stack.push(reader.string(reader.byte()));
break;
case OpCode.UNICODE:
stack.push(reader.line());
break;
case OpCode.BINUNICODE:
stack.push(reader.string(reader.uint32(), 'utf-8'));
break;
case OpCode.SHORT_BINUNICODE:
stack.push(reader.string(reader.byte(), 'utf-8'));
break;
case OpCode.BUILD: {
const state = stack.pop();
let obj = stack.pop();
if (obj.__setstate__) {
if (obj.__setstate__.__call__) {
obj.__setstate__.__call__([ obj, state ]);
}
else {
obj.__setstate__(state);
}
}
else if (ArrayBuffer.isView(state) || Object(state) !== state) {
obj.__state__ = state;
}
else if (obj instanceof Map) {
for (const key in state) {
obj.set(key, state[key]);
}
}
else {
Object.assign(obj, state);
}
if (obj.__read__) {
obj = obj.__read__(this);
}
stack.push(obj);
break;
}
case OpCode.MARK:
marker.push(stack);
stack = [];
break;
case OpCode.NEWTRUE:
stack.push(true);
break;
case OpCode.NEWFALSE:
stack.push(false);
break;
case OpCode.LONG1: {
const data = reader.read(reader.byte());
let number = 0;
switch (data.length) {
case 0: number = 0; break;
case 1: number = data[0]; break;
case 2: number = data[1] << 8 | data[0]; break;
case 3: number = data[2] << 16 | data[1] << 8 | data[0]; break;
case 4: number = data[3] << 24 | data[2] << 16 | data[1] << 8 | data[0]; break;
case 5: number = data[4] * 0x100000000 + ((data[3] << 24 | data[2] << 16 | data[1] << 8 | data[0]) >>> 0); break;
default: number = Array.prototype.slice.call(data, 0); break;
}
stack.push(number);
break;
}
case OpCode.LONG4:
// TODO decode LONG4
stack.push(reader.read(reader.uint32()));
break;
case OpCode.TUPLE1:
stack.push([ stack.pop() ]);
break;
case OpCode.TUPLE2: {
const b = stack.pop();
const a = stack.pop();
stack.push([ a, b ]);
break;
}
case OpCode.TUPLE3: {
const c = stack.pop();
const b = stack.pop();
const a = stack.pop();
stack.push([ a, b, c ]);
break;
}
case OpCode.MEMOIZE:
memo.set(memo.size, stack[stack.length - 1]);
break;
case OpCode.FRAME:
reader.read(8);
break;
case OpCode.BYTEARRAY8: {
stack.push(reader.read(reader.int64()));
break;
}
case OpCode.NONE:
stack.push(null);
break;
case OpCode.STOP:
return stack.pop();
default:
throw new python.Error('Unknown opcode ' + opcode + ' at position ' + (reader.position - 1).toString() + '.');
}
}
throw new python.Error('Unexpected end of file.');
}
read(size) {
return this._reader.read(size);
}
stream(size) {
return this._reader.stream(size);
}
int32() {
return this._reader.int32();
}
int64() {
return this._reader.int64();
}
unescape(token, size) {
const length = token.length;
const a = new Uint8Array(length);
if (size && size == length) {
for (let p = 0; p < size; p++) {
a[p] = token.charCodeAt(p);
}
return a;
}
let i = 0;
let o = 0;
while (i < length) {
let c = token.charCodeAt(i++);
if (c !== 0x5C || i >= length) {
a[o++] = c;
}
else {
c = token.charCodeAt(i++);
switch (c) {
case 0x27: a[o++] = 0x27; break; // '
case 0x5C: a[o++] = 0x5C; break; // \\
case 0x22: a[o++] = 0x22; break; // "
case 0x72: a[o++] = 0x0D; break; // \r
case 0x6E: a[o++] = 0x0A; break; // \n
case 0x74: a[o++] = 0x09; break; // \t
case 0x62: a[o++] = 0x08; break; // \b
case 0x58: // x
case 0x78: { // X
const xsi = i - 1;
const xso = o;
for (let xi = 0; xi < 2; xi++) {
if (i >= length) {
i = xsi;
o = xso;
a[o] = 0x5c;
break;
}
let xd = token.charCodeAt(i++);
xd = xd >= 65 && xd <= 70 ? xd - 55 : xd >= 97 && xd <= 102 ? xd - 87 : xd >= 48 && xd <= 57 ? xd - 48 : -1;
if (xd === -1) {
i = xsi;
o = xso;
a[o] = 0x5c;
break;
}
a[o] = a[o] << 4 | xd;
}
o++;
break;
}
default:
if (c < 48 || c > 57) { // 0-9
a[o++] = 0x5c;
a[o++] = c;
}
else {
i--;
const osi = i;
const oso = o;
for (let oi = 0; oi < 3; oi++) {
if (i >= length) {
i = osi;
o = oso;
a[o] = 0x5c;
break;
}
const od = token.charCodeAt(i++);
if (od < 48 || od > 57) {
i = osi;
o = oso;
a[o] = 0x5c;
break;
}
a[o] = a[o] << 3 | od - 48;
}
o++;
}
break;
}
}
}
return a.slice(0, o);
}
};
// https://svn.python.org/projects/python/trunk/Lib/pickletools.py
// https://github.com/python/cpython/blob/master/Lib/pickle.py
python.Unpickler.OpCode = {
MARK: 40, // '('
EMPTY_TUPLE: 41, // ')'
STOP: 46, // '.'
POP: 48, // '0'
POP_MARK: 49, // '1'
DUP: 50, // '2'
BINBYTES: 66, // 'B' (Protocol 3)
SHORT_BINBYTES: 67, // 'C' (Protocol 3)
FLOAT: 70, // 'F'
BINFLOAT: 71, // 'G'
INT: 73, // 'I'
BININT: 74, // 'J'
BININT1: 75, // 'K'
LONG: 76, // 'L'
BININT2: 77, // 'M'
NONE: 78, // 'N'
PERSID: 80, // 'P'
BINPERSID: 81, // 'Q'
REDUCE: 82, // 'R'
STRING: 83, // 'S'
BINSTRING: 84, // 'T'
SHORT_BINSTRING: 85, // 'U'
UNICODE: 86, // 'V'
BINUNICODE: 88, // 'X'
EMPTY_LIST: 93, // ']'
APPEND: 97, // 'a'
BUILD: 98, // 'b'
GLOBAL: 99, // 'c'
DICT: 100, // 'd'
APPENDS: 101, // 'e'
GET: 103, // 'g'
BINGET: 104, // 'h'
INST: 105, // 'i'
LONG_BINGET: 106, // 'j'
LIST: 108, // 'l'
OBJ: 111, // 'o'
PUT: 112, // 'p'
BINPUT: 113, // 'q'
LONG_BINPUT: 114, // 'r'
SETITEM: 115, // 's'
TUPLE: 116, // 't'
SETITEMS: 117, // 'u'
EMPTY_DICT: 125, // '}'
PROTO: 128,
NEWOBJ: 129,
TUPLE1: 133, // '\x85'
TUPLE2: 134, // '\x86'
TUPLE3: 135, // '\x87'
NEWTRUE: 136, // '\x88'
NEWFALSE: 137, // '\x89'
LONG1: 138, // '\x8a'
LONG4: 139, // '\x8b'
SHORT_BINUNICODE: 140, // '\x8c' (Protocol 4)
BINUNICODE8: 141, // '\x8d' (Protocol 4)
BINBYTES8: 142, // '\x8e' (Protocol 4)
EMPTY_SET: 143, // '\x8f' (Protocol 4)
ADDITEMS: 144, // '\x90' (Protocol 4)
FROZENSET: 145, // '\x91' (Protocol 4)
NEWOBJ_EX: 146, // '\x92' (Protocol 4)
STACK_GLOBAL: 147, // '\x93' (Protocol 4)
MEMOIZE: 148, // '\x94' (Protocol 4)
FRAME: 149, // '\x95' (Protocol 4)
BYTEARRAY8: 150, // '\x96' (Protocol 5)
NEXT_BUFFER: 151, // '\x97' (Protocol 5)
READONLY_BUFFER: 152 // '\x98' (Protocol 5)
};
python.Unpickler.BinaryReader = class {
constructor(buffer) {
this._buffer = buffer;
this._length = buffer.length;
this._position = 0;
this._view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
this._utf8Decoder = new TextDecoder('utf-8');
this._asciiDecoder = new TextDecoder('ascii');
}
get position() {
return this._position;
}
get length() {
return this._length;
}
seek(position) {
this._position = position >= 0 ? position : this._length + position;
if (this._position > this._buffer.length) {
throw new Error('Expected ' + (this._position - this._buffer.length) + ' more bytes. The file might be corrupted. Unexpected end of file.');
}
}
skip(offset) {
this._position += offset;
if (this._position > this._buffer.length) {
throw new python.Error('Expected ' + (this._position - this._buffer.length) + ' more bytes. The file might be corrupted. Unexpected end of file.');
}
}
stream(length) {
const buffer = this.read(length);
return new python.Unpickler.BinaryReader(buffer);
}
peek(length) {
const position = this._position;
length = length !== undefined ? length : this._length - this._position;
this.skip(length);
const end = this._position;
this.skip(-length);
if (position === 0 && length === this._length) {
return this._buffer;
}
return this._buffer.subarray(position, end);
}
read(length) {
const position = this._position;
length = length !== undefined ? length : this._length - this._position;
this.skip(length);
if (position === 0 && length === this._length) {
return this._buffer;
}
return this._buffer.subarray(position, this._position);
}
byte() {
const position = this._position;
this.skip(1);
return this._view.getUint8(position);
}
uint16() {
const position = this._position;
this.skip(2);
return this._view.getUint16(position, true);
}
int32() {
const position = this._position;
this.skip(4);
return this._view.getInt32(position, true);
}
uint32() {
const position = this._position;
this.skip(4);
return this._view.getUint32(position, true);
}
int64() {
const position = this._position;
this.skip(8);
return this._view.getInt64(position, true).toNumber();
}
float32() {
const position = this._position;
this.skip(4);
return this._view.getFloat32(position, true);
}
float64() {
const position = this._position;
this.skip(8);
return this._view.getFloat64(position, true);
}
string(size, encoding) {
const data = this.read(size);
return (encoding == 'utf-8') ?
this._utf8Decoder.decode(data) :
this._asciiDecoder.decode(data);
}
line() {
const index = this._buffer.indexOf(0x0A, this._position);
if (index == -1) {
throw new python.Error("Could not find end of line.");
}
const size = index - this._position;
const text = this.string(size, 'ascii');
this.skip(1);
return text;
}
};
python.Unpickler.StreamReader = class {
constructor(stream) {
this._stream = stream;
this._length = stream.length;
this._position = 0;
this._utf8Decoder = new TextDecoder('utf-8');
this._asciiDecoder = new TextDecoder('ascii');
}
get position() {
return this._position;
}
get length() {
return this._length;
}
seek(position) {
this._stream.seek(position);
this._position = this._stream.position;
}
skip(offset) {
this._position += offset;
if (this._position > this._length) {
throw new python.Error('Expected ' + (this._position - this._length) + ' more bytes. The file might be corrupted. Unexpected end of file.');
}
}
stream(length) {
this._stream.seek(this._position);
this.skip(length);
return this._stream.stream(length);
}
peek(length) {
this._stream.seek(this._position);
return this._stream.peek(length);
}
read(length) {
this._stream.seek(this._position);
this.skip(length);
return this._stream.read(length);
}
byte() {
const position = this._fill(1);
return this._view.getUint8(position);
}
uint16() {
const position = this._fill(2);
return this._view.getUint16(position, true);
}
int32() {
const position = this._fill(4);
return this._view.getInt32(position, true);
}
uint32() {
const position = this._fill(4);
return this._view.getUint32(position, true);
}
int64() {
const position = this._fill(8);
return this._view.getInt64(position, true).toNumber();
}
float32() {
const position = this._fill(4);
return this._view.getFloat32(position, true);
}
float64() {
const position = this._fill(8);
return this._view.getFloat64(position, true);
}
string(size, encoding) {
const data = this.read(size);
return (encoding == 'utf-8') ?
this._utf8Decoder.decode(data) :
this._asciiDecoder.decode(data);
}
line() {
let position = this._fill(0);
let index = this._buffer.indexOf(0x0A, position);
if (index == -1) {
const size = Math.min(0x1000000, this._stream.length - this._position);
this._fill(size);
this.skip(-size);
position = this._fill(0);
index = this._buffer.indexOf(0x0A, position);
if (index == -1) {
throw new python.Error("Could not find end of line.");
}
}
const size = index - position;
const text = this.string(size, 'ascii');
this.skip(1);
return text;
}
_fill(length) {
if (this._position + length > this._length) {
throw new Error('Expected ' + (this._position + length - this._length) + ' more bytes. The file might be corrupted. Unexpected end of file.');
}
if (!this._buffer || this._position < this._offset || this._position + length > this._offset + this._buffer.length) {
this._offset = this._position;
this._stream.seek(this._offset);
this._buffer = this._stream.read(Math.min(0x10000000, this._length - this._offset));
this._view = new DataView(this._buffer.buffer, this._buffer.byteOffset, this._buffer.byteLength);
}
const position = this._position;
this._position += length;
return position - this._offset;
}
};
python.Error = class extends Error {
constructor(message) {
super(message);
this.name = 'Error loading Python module.';
}
};
if (typeof module !== 'undefined' && typeof module.exports === 'object') {
module.exports.Execution = python.Execution;
module.exports.Unpickler = python.Unpickler;
}
Reference in New Issue View Git Blame Copy Permalink