## @package generator # Module caffe2.python.docs.generator import argparse import os from caffe2.python import core, workspace from caffe2.python.docs.formatter import Markdown OpSchema = workspace.C.OpSchema class DocUploader(object): def __init__(self): pass def upload(self, text): pass class DocGenerator(object): def __init__(self, formatter, uploader): self.formatter = formatter self.uploader = uploader self.content_body = "" def create_body(self): pass def update(self): self.uploader.upload(self.content_body) class OpDocGenerator(DocGenerator): def getOperatorDoc(self, name, schema, priority): return OperatorDoc(name, schema, priority) def getOperatorEngine(self, name): return OperatorEngine(name) def getOperators(self): # map: op_name -> operator self.operators = {} # map: op_name -> [engine, engine] self.engines = {} def filePriority(x): if x == "caffe2/caffe2/operators": return 0 if 'contrib' in x.split('/'): return 2 if 'experiments' in x.split('/'): return 3 return 1 for name in core._GetRegisteredOperators(): schema = OpSchema.get(name) if schema: priority = filePriority(os.path.dirname(schema.file)) operator = self.getOperatorDoc(name, schema, priority) self.operators[name] = operator # Engine elif name.find("_ENGINE_") != -1: engine = self.getOperatorEngine(name) if engine.base_op_name in self.engines: self.engines[engine.base_op_name].append(engine) else: self.engines[engine.base_op_name] = [engine] # No schema else: priority = 4 self.operators[name] = self.getOperatorDoc(name, schema, priority) for name, engines in self.engines.items(): if name in self.operators: self.operators[name].addEngines(engines) # Generate a sorted list of operators return sorted( self.operators.values(), key=lambda op: (op.priority, op.name) ) def createBody(self): operators = self.getOperators() for operator in operators: operator.generateSchema(self.formatter) self.content_body += self.formatter.dump() class OperatorEngine(object): def __init__(self, name): self.op_name = name self.base_op_name, self.engine = name.split("_ENGINE_", 1) def getDeviceImpl(self): deviceImplList = [] for device, impl in [('CPU', OpSchema.get_cpu_impl(self.op_name)), ('CUDA', OpSchema.get_cuda_impl(self.op_name))]: if not impl: continue deviceImplList.append((device, impl)) return deviceImplList def generateDoc(self, formatter): for device, impl in self.getDeviceImpl(): formatter.addLine( '{engine} on {device}: {impl}'.format(engine=self.engine, device=device, impl=impl)) class OperatorDoc(object): def __init__(self, name, schema, priority): self.name = name self.schema = schema self.priority = priority print("Gathering docs for {}...".format(self.name)) self.engines = [] def addEngines(self, engines): self.engines = engines def generateDoc(self, formatter): if self.schema.doc: formatter.parseAndAdd(self.schema.doc) formatter.addLinebreak() else: formatter.addLine("No documentation yet.") def generateTable(self, formatter, tuples, title_row, title): if tuples: if title: formatter.addHeader(title, 3) table = [] if title_row: table = [title_row] for name, doc in tuples: table.append([name, doc or '']) formatter.addTable(table, (table == [])) def generateInterface(self, formatter): def makeDesc(title, args): f = formatter.clone() f.addEmphasis(title, 1) out = [(f.dump(), '')] for arg in args: f = formatter.clone() if isinstance(arg, tuple): name = arg[0] if len(arg) > 1: description = arg[1] or '' else: description = '' else: name = arg.name description = arg.description or '' f.addCode(name, inline=True) out.append((f.dump(), description or '')) return out tuples = [] if self.schema.args: tuples += makeDesc('Arguments', self.schema.args) if self.schema.input_desc: tuples += makeDesc('Inputs', self.schema.input_desc) if self.schema.output_desc: tuples += makeDesc('Outputs', self.schema.output_desc) self.generateTable(formatter, tuples, None, 'Interface') print("Generated interface for {}".format(self.name)) def generateCodeLink(self, formatter): formatter.addHeader("Code", 3) formatter.addLinebreak() formatter.addCodeLink(self.schema.file) def getInfo(self, formatter, name, impl): pass def generateDevices(self, formatter): formatter.addHeader("Devices", 3) devices = [ self.getInfo(formatter, 'CPU', OpSchema.get_cpu_impl(self.name)), self.getInfo(formatter, 'GPU', OpSchema.get_cuda_impl(self.name)), ] formatter.addList([i for i in devices if i]) def generateEngines(self, formatter): if not len(self.engines): return formatter.addHeader("Engines", 3) for engine in self.engines: engine.generateDoc(formatter) def generateSchema(self, formatter): formatter.addHeader(self.name, 2) if self.schema: self.generateDoc(formatter) self.generateInterface(formatter) self.generateCodeLink(formatter) self.generateDevices(formatter) self.generateEngines(formatter) formatter.addBreak() else: formatter.addLine("No schema documented yet.") self.generateDevices(formatter) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Operators catalog generator.") parser.add_argument('catalog_path', type=str, help='operators-catalogue.md to write out to') args = parser.parse_args() with open(args.catalog_path, 'w') as fp: ops = OpDocGenerator(Markdown(), DocUploader()) ops.createBody() fp.write(ops.content_body)