diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index d79b86c36b..f23a1b3047 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -218,7 +218,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) COMPONENT "docs" OPTIONAL ) - if(BUILD_opencv_python2) + if(PYTHON2_EXECUTABLE) add_custom_target(doxygen_python COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" "python" DEPENDS "${doxygen_result}" gen_opencv_python2 @@ -226,7 +226,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND) add_custom_target(doxygen DEPENDS doxygen_cpp doxygen_python ) - elseif(BUILD_opencv_python3) + elseif(PYTHON3_EXECUTABLE) add_custom_target(doxygen_python COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" "python" DEPENDS "${doxygen_result}" gen_opencv_python3 diff --git a/doc/tools/add_signatures.py b/doc/tools/add_signatures.py index 95b57c2779..778dd33570 100644 --- a/doc/tools/add_signatures.py +++ b/doc/tools/add_signatures.py @@ -12,10 +12,15 @@ TODO: http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41 """ from __future__ import print_function -import os -import re import sys +sys.dont_write_bytecode = True # Don't generate .pyc files / __pycache__ directories + +import os +from pprint import pprint +import re import logging +import json + import html_functions import doxygen_scan @@ -23,37 +28,23 @@ loglevel=os.environ.get("LOGLEVEL", None) if loglevel: logging.basicConfig(level=loglevel) - ROOT_DIR = sys.argv[1] PYTHON_SIGNATURES_FILE = sys.argv[2] -JAVA_PYTHON = sys.argv[3] +JAVA_OR_PYTHON = sys.argv[3] ADD_JAVA = False ADD_PYTHON = False -if JAVA_PYTHON == "python": +if JAVA_OR_PYTHON == "python": ADD_PYTHON = True -import json python_signatures = dict() with open(PYTHON_SIGNATURES_FILE, "rt") as f: python_signatures = json.load(f) print("Loaded Python signatures: %d" % len(python_signatures)) -# only name -> class -# name and ret -> constant -# name, ret, arg-> function / class method - -class Configuration(): - def __init__(self): - self.ADD_PYTHON = ADD_PYTHON - self.python_signatures = python_signatures - self.ADD_JAVA = ADD_JAVA - -config = Configuration() - import xml.etree.ElementTree as ET root = ET.parse(ROOT_DIR + 'opencv.tag') -files_dict = dict() +files_dict = {} # constants and function from opencv.tag namespaces = root.findall("./compound[@kind='namespace']") @@ -61,41 +52,48 @@ namespaces = root.findall("./compound[@kind='namespace']") for ns in namespaces: ns_name = ns.find("./name").text #print('NS: {}'.format(ns_name)) - - files_dict = doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict) - files_dict = doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict) + doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict) + doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict) # class methods from opencv.tag classes = root.findall("./compound[@kind='class']") #print("Found {} classes".format(len(classes))) for c in classes: c_name = c.find("./name").text - name = ns_name + '::' + c_name file = c.find("./filename").text - #print('Class: {} => {}'.format(name, file)) - files_dict = doxygen_scan.scan_class_methods(c, c_name, files_dict) + #print('Class: {} => {}'.format(c_name, file)) + doxygen_scan.scan_class_methods(c, c_name, files_dict) + +print('Doxygen files to scan: %s' % len(files_dict)) + +files_processed = 0 +files_skipped = 0 +symbols_processed = 0 -# test for file in files_dict: - soup = html_functions.load_html_file(ROOT_DIR + file) - if file == "dd/d9e/classcv_1_1VideoWriter.html":#"d4/d86/group__imgproc__filter.html":#"d4/d86/group__imgproc__filter.html": - anchor_list = files_dict[file] - counter = 0 - anchor_tmp_list = [] - for anchor in anchor_list: - counter += 1 - # if the next anchor shares the same C++ name (= same method/function), join them together - if counter < len(anchor_list) and anchor_list[counter].cppname == anchor.cppname: - anchor_tmp_list.append(anchor) - continue - else: - anchor_tmp_list.append(anchor) - # check if extists a python equivalent signature - for signature in python_signatures: # signature is a key with the C++ name - if signature == anchor.cppname: # if available name in python - # they should also have the same type - soup = html_functions.append_python_signature(python_signatures[signature], anchor_tmp_list, soup) - #print(signature) - # reset anchor temporary list - anchor_tmp_list[:] = [] - html_functions.update_html(ROOT_DIR + file, soup) + #if file != "dd/d9e/classcv_1_1VideoWriter.html": + #if file != "d4/d86/group__imgproc__filter.html": + #if file != "df/dfb/group__imgproc__object.html": + # continue + #print('File: ' + file) + + anchor_list = files_dict[file] + active_anchors = [a for a in anchor_list if a.cppname in python_signatures] + if len(active_anchors) == 0: # no linked Python symbols + #print('Skip: ' + file) + files_skipped = files_skipped + 1 + continue + + active_anchors_dict = {a.anchor: a for a in active_anchors} + if len(active_anchors_dict) != len(active_anchors): + logging.info('Duplicate entries detected: %s -> %s (%s)' % (len(active_anchors), len(active_anchors_dict), file)) + + files_processed = files_processed + 1 + + #pprint(active_anchors) + symbols_processed = symbols_processed + len(active_anchors_dict) + + logging.info('File: %r' % file) + html_functions.insert_python_signatures(python_signatures, active_anchors_dict, ROOT_DIR + file) + +print('Done (processed files %d, symbols %d, skipped %d files)' % (files_processed, symbols_processed, files_skipped)) diff --git a/doc/tools/doxygen_scan.py b/doc/tools/doxygen_scan.py index 4567723302..a1ce60f618 100644 --- a/doc/tools/doxygen_scan.py +++ b/doc/tools/doxygen_scan.py @@ -1,20 +1,20 @@ -class Anchor(object): - anchor = "" - type = "" - cppname = "" +import traceback +class Symbol(object): def __init__(self, anchor, type, cppname): self.anchor = anchor self.type = type self.cppname = cppname + #if anchor == 'ga586ebfb0a7fb604b35a23d85391329be': + # print(repr(self)) + # traceback.print_stack() + + def __repr__(self): + return '%s:%s@%s' % (self.type, self.cppname, self.anchor) def add_to_file(files_dict, file, anchor): - if file in files_dict: - # if that file already exists as a key in the dictionary - files_dict[file].append(anchor) - else: - files_dict[file] = [anchor] - return files_dict + anchors = files_dict.setdefault(file, []) + anchors.append(anchor) def scan_namespace_constants(ns, ns_name, files_dict): @@ -25,8 +25,7 @@ def scan_namespace_constants(ns, ns_name, files_dict): file = c.find("./anchorfile").text anchor = c.find("./anchor").text #print(' CONST: {} => {}#{}'.format(name, file, anchor)) - files_dict = add_to_file(files_dict, file, Anchor(anchor, "const", name)) - return files_dict + add_to_file(files_dict, file, Symbol(anchor, "const", name)) def scan_namespace_functions(ns, ns_name, files_dict): functions = ns.findall("./member[@kind='function']") @@ -36,8 +35,7 @@ def scan_namespace_functions(ns, ns_name, files_dict): file = f.find("./anchorfile").text anchor = f.find("./anchor").text #print(' FN: {} => {}#{}'.format(name, file, anchor)) - files_dict = add_to_file(files_dict, file, Anchor(anchor, "fn", name)) - return files_dict + add_to_file(files_dict, file, Symbol(anchor, "fn", name)) def scan_class_methods(c, c_name, files_dict): methods = c.findall("./member[@kind='function']") @@ -47,5 +45,4 @@ def scan_class_methods(c, c_name, files_dict): file = m.find("./anchorfile").text anchor = m.find("./anchor").text #print(' Method: {} => {}#{}'.format(name, file, anchor)) - files_dict = add_to_file(files_dict, file, Anchor(anchor, "method", name)) - return files_dict + add_to_file(files_dict, file, Symbol(anchor, "method", name)) diff --git a/doc/tools/html_functions.py b/doc/tools/html_functions.py index 9f9182a27b..c3fd3bd8e9 100644 --- a/doc/tools/html_functions.py +++ b/doc/tools/html_functions.py @@ -1,7 +1,10 @@ from __future__ import print_function +import sys + import logging import os from pprint import pprint +import traceback try: import bs4 @@ -13,195 +16,110 @@ except ImportError: def load_html_file(file_dir): """ Uses BeautifulSoup to load an html """ - with open(file_dir) as fp: + with open(file_dir, 'rb') as fp: soup = BeautifulSoup(fp, 'html.parser') return soup -def add_item(soup, new_row, is_parameter, text): - """ Adds a new html tag for the table with the signature """ - new_item = soup.new_tag('td') - if is_parameter: - new_item = soup.new_tag('td', **{'class': 'paramname'}) - new_item.append(text) - new_row.append(new_item) - return new_row, soup - -def add_signature_to_table(soup, tmp_row, signature, language, type): - """ Add a signature to an html table""" - new_item = soup.new_tag('td', style="padding-left: 0.5cm;") - - if str(signature.get('ret', None)) != "None": - new_item.append(signature.get('ret') + ' =') - tmp_row.append(new_item) - - tmp_name = signature.get('name', None) - if type is not "method": - tmp_name = "cv2." + tmp_name - else: - tmp_name = "obj." + tmp_name - tmp_row, soup = add_item(soup, tmp_row, False, tmp_name + '(') - tmp_row, soup = add_item(soup, tmp_row, True, signature['arg']) - tmp_row, soup = add_item(soup, tmp_row, False, ')') - return tmp_row, soup - - -def new_line(soup, tmp_table, new_row): - """ Adds a new line to the html table """ - tmp_table.append(new_row) - new_row = soup.new_tag('tr') - return new_row, soup - - -def add_bolded(soup, new_row, text): - """ Adds bolded text to the table """ - new_item = soup.new_tag('th', style="text-align:left") - new_item.append(text) - new_row.append(new_item) - return new_row, soup - - -def create_description(soup, language, signatures, type): - """ Insert the new Python / Java table after the current html c++ table """ - assert signatures - tmp_table = soup.new_tag('table') - new_row = soup.new_tag('tr') - new_row, soup = add_bolded(soup, new_row, language) - new_row, soup = new_line(soup, tmp_table, new_row) - for s in signatures: - new_row, soup = new_line(soup, tmp_table, new_row) - new_row, soup = add_signature_to_table(soup, new_row, s, language, type) - new_row, soup = new_line(soup, tmp_table, new_row) - return tmp_table, soup - - -def get_anchor_list(anchor, soup): - a_list = [] - # go through all the links - for a in soup.find_all('a', href=True): - # find links with the same anchor - last_part_of_link = a['href'].rsplit('#', 1)[-1] - if last_part_of_link == anchor: - a_list.append(a) - return a_list - -def is_static_method(element): - if element.name == "table": - tmp_element = element.find('td', {'class': 'memname'}) - if tmp_element is not None: - if 'static' in tmp_element.text: - return True - else: - if element['class'][0] == 'memItemRight': - if "static" in element.previousSibling.text: - return True - return False - -def append_python_signatures_to_table(soup, signatures, table, type): - if type == "method": - if is_static_method(table): - type = "static" + type - description, soup = create_description(soup, "Python:", signatures, type) - description['class'] = 'python_language' - soup = insert_or_replace(soup, table, description, "table", "python_language") - return soup - -def get_heading_text(a): - str = "" - element = a.parent - if element is not None: - childs = element.find_all('a') - # the anchor should not be an argument of a function / method - if childs.index(a) is not 0: - return str - element = element.parent - if element is not None: - if element.has_attr('class'): - tmp_class = element["class"][0] - if "memitem:" in tmp_class and "python" not in tmp_class: - str = element.parent.find("tr").text - return str - -def insert_or_replace(soup, element, description, tag_name, tag_class): - old = element.next_sibling - if old is not None: - if old.name != tag_name: - old = None - elif not tag_class in old.get('class', []): - old = None - # if already existed replace with the new - if old is None: - element.insert_after(description) - else: - old.replace_with(description) - return soup - -def new_heading_td(soup, s, href, type): - if href is None: - attrs = {'class': 'memItemLeft', 'valign': 'top', 'align': 'right'} - new_td = soup.new_tag('td', **attrs) - new_td.append(str(s.get('ret', None))) - else: - attrs = {'class': 'memItemRight', 'valign': 'bottom'} - new_td = soup.new_tag('td', **attrs) - - # make the function name linkable - attrs_a = {'class': 'el', 'href': href} - new_a = soup.new_tag('a', **attrs_a) - tmp_name = str(s.get('name', None)) - if type is not "method": - tmp_name = "cv2." + tmp_name - else: - tmp_name = "obj." + tmp_name - new_a.append(tmp_name) - new_td.append(new_a) - - new_td.append("(" + s['arg'] +")") - return soup, new_td - -def append_python_signatures_to_heading(soup, signatures, element, href, type): - if type == "method": - if is_static_method(element): - type = "static" + type - for s in signatures: - attrs = {'class': 'memitem:python'} - new_tr = soup.new_tag('tr', **attrs) - - soup, new_td_left = new_heading_td(soup, s, None, type) - new_tr.append(new_td_left) - - soup, new_td_right = new_heading_td(soup, s, href, type) - new_tr.append(new_td_right) - - soup = insert_or_replace(soup, element, new_tr, "tr", "memitem:python") - return soup - -def append_python_signature(function_variants, anchor_list, soup): - type = anchor_list[0].type - if type == "method" or type == "fn": - if len(anchor_list) == 1: - tmp_anchor = anchor_list[0].anchor - a_list = get_anchor_list(tmp_anchor, soup) - for a in a_list: - if a['href'] == "#" + tmp_anchor: - tmp_element = a.parent - # ignore the More... link - if tmp_element is None or tmp_element['class'][0] == 'mdescRight': - continue - # Function Documentation (tables) - table = a.findNext('table') - if table is not None: - soup = append_python_signatures_to_table(soup, function_variants, table, type) - else: - str = get_heading_text(a) - if "Functions" in str: - soup = append_python_signatures_to_heading(soup, function_variants, a.parent, a['href'], type) - return soup - def update_html(file, soup): - tmp_str = str(soup) - if os.name == 'nt': # if Windows - with open(file, "wb") as tmp_file: - tmp_file.write(tmp_str.encode("ascii","ignore")) + s = str(soup) + if os.name == 'nt' or sys.version_info[0] == 3: # if Windows + s = s.encode('utf-8', 'ignore') + with open(file, 'wb') as f: + f.write(s) + + +def insert_python_signatures(python_signatures, symbols_dict, filepath): + soup = load_html_file(filepath) + entries = soup.find_all(lambda tag: tag.name == "a" and tag.has_attr('id')) + for e in entries: + anchor = e['id'] + if anchor in symbols_dict: + s = symbols_dict[anchor] + logging.info('Process: %r' % s) + if s.type == 'fn' or s.type == 'method': + process_fn(soup, e, python_signatures[s.cppname], s) + elif s.type == 'const': + process_const(soup, e, python_signatures[s.cppname], s) + else: + logging.error('unsupported type: %s' % s); + + update_html(filepath, soup) + + +def process_fn(soup, anchor, python_signature, symbol): + try: + r = anchor.find_next_sibling(class_='memitem').find(class_='memproto').find('table') + insert_python_fn_signature(soup, r, python_signature, symbol) + except: + logging.error("Can't process: %s" % symbol) + traceback.print_exc() + pprint(anchor) + + +def process_const(soup, anchor, python_signature, symbol): + try: + #pprint(anchor.parent) + description = append(soup.new_tag('div', **{'class' : ['python_language']}), + 'Python: ' + python_signature[0]['name']) + old = anchor.find_next_sibling('div', class_='python_language') + if old is None: + anchor.parent.append(description) + else: + old.replace_with(description) + #pprint(anchor.parent) + except: + logging.error("Can't process: %s" % symbol) + traceback.print_exc() + pprint(anchor) + + +def insert_python_fn_signature(soup, table, variants, symbol): + description = create_python_fn_description(soup, variants) + description['class'] = 'python_language' + soup = insert_or_replace(table, description, 'table', 'python_language') + return soup + + +def create_python_fn_description(soup, variants): + language = 'Python:' + table = soup.new_tag('table') + heading_row = soup.new_tag('th') + table.append( + append(soup.new_tag('tr'), + append(soup.new_tag('th', colspan=999, style="text-align:left"), language))) + for v in variants: + #logging.debug(v) + add_signature_to_table(soup, table, v, language, type) + #print(table) + return table + + +def add_signature_to_table(soup, table, signature, language, type): + """ Add a signature to an html table""" + row = soup.new_tag('tr') + row.append(soup.new_tag('td', style='width: 20px;')) + + if 'ret' in signature: + row.append(append(soup.new_tag('td'), signature['ret'])) + row.append(append(soup.new_tag('td'), '=')) else: - with open(file, "w") as tmp_file: - tmp_file.write(tmp_str) + row.append(soup.new_tag('td')) # return values + row.append(soup.new_tag('td')) # '=' + + row.append(append(soup.new_tag('td'), signature['name'] + '(')) + row.append(append(soup.new_tag('td', **{'class': 'paramname'}), signature['arg'])) + row.append(append(soup.new_tag('td'), ')')) + table.append(row) + + +def append(target, obj): + target.append(obj) + return target + + +def insert_or_replace(element_before, new_element, tag, tag_class): + old = element_before.find_next_sibling(tag, class_=tag_class) + if old is None: + element_before.insert_after(new_element) + else: + old.replace_with(new_element)