diff --git a/admin/sitegen-lib/afpstats.py b/admin/sitegen-lib/afpstats.py deleted file mode 100755 --- a/admin/sitegen-lib/afpstats.py +++ /dev/null @@ -1,129 +0,0 @@ -from datetime import datetime -import os.path -import re - -import pytz - -#TODO: script relies on checking paths, can this be broken? - -def normpath(path, *paths): - try: - return os.path.abspath(os.path.join(path, *paths)) - # only happens in python version < 3, non ascii cant be part of a path - except UnicodeDecodeError: - return None - -class afp_author(object): - """An AFP author has a name and a web or mail address""" - def __init__(self, name, address): - self.name = name - self.address = address - self.articles = set() - - def __eq__(self, other): - return self.name == other.name - - def __hash__(self): - return hash(self.name) - -class afp_entry(object): - """An AFP entry consists of metadata (date, authors, etc), used imports, - used library imports, paths to thys files and which AFP entries import - it. - Not all of this data is present after initialization. See also class - afp_dict. - It still relies on information created by the entries-dict in sitegen.py. - """ - def __init__(self, name, entry_dict, afp_dict, no_index=False): - self.name = name - self.afp_dict = afp_dict - self.path = os.path.join(self.afp_dict.path, self.name) - self.title = entry_dict['title'] - #TODO: fix author generation, Contributors??? - self.authors = [] - for name, _address in entry_dict['author']: - self.authors.append(afp_dict.authors[name]) - if not no_index: - afp_dict.authors[name].articles.add(self) - self.publish_date = datetime.strptime(entry_dict['date'], "%Y-%m-%d") - #add UTC timezone to date - self.publish_date = self.publish_date.replace(tzinfo=pytz.UTC) - self.abstract = entry_dict['abstract'] - self.license = entry_dict['license'] - self.releases = list(entry_dict['releases'].items()) - self.contributors = (entry_dict['contributors'] - if entry_dict['contributors'][0][0] else []) - self.extra = entry_dict['extra'] - self.thys = set() - self.status = None - for root, _dirnames, filenames in os.walk(self.path): - for f in filenames: - if f.endswith(".thy"): - self.thys.add(os.path.join(self.path, root, f)) - - def __str__(self): - return self.name - - def __repr__(self): - return self.name - - def add_loc(self): - """Count non empty lines in thy-files""" - self.loc = 0 - for t in self.thys: - with open(t, 'r') as f: - for l in f: - if l.strip(): - self.loc += 1 - - def add_number_of_lemmas(self): - """Count number of lemmas, theorems and corollarys""" - self.lemmas = 0 - for t in self.thys: - with open(t, 'r') as f: - for l in f: - if l.startswith("lemma") or l.startswith("corollary") or \ - l.startswith("theorem"): - self.lemmas += 1 - - -class afp_dict(dict): - """The afp_dict contains all afp_entry(s) and a list of afp_author(s). - To create import/export data for all afp_entrys call build_stats(). - """ - def __init__(self, entries, afp_thys_path, deps_dict, *args): - dict.__init__(self, *args) - self.path = normpath(afp_thys_path) - self.authors = dict() - # Extra dict for entries which don't show up in index and statistics - #TODO: document how it works, improve how it works - self.no_index = dict() - for entry in entries.values(): - for name, address in entry['author']: - self.authors[name] = afp_author(name, address) - for name, entry in entries.items(): - if 'extra' in entry and 'no-index' in entry['extra']: - self.no_index[name] = afp_entry(name, entry, self, - no_index=True) - else: - self[name] = afp_entry(name, entry, self) - for name in self.no_index: - del entries[name] - # all_thys is a dict which maps a thy file to its corresponding AFP - # entry - self.all_thys = dict() - for a in self: - for t in self[a].thys: - self.all_thys[t] = self[a] - for k, a in self.items(): - a.imports = set([self[e] for e in deps_dict[k]['afp_deps']]) - a.lib_imports = set(deps_dict[k]['distrib_deps']) - - def build_stats(self): - for _k, a in self.items(): - a.add_loc() - a.add_number_of_lemmas() - a.used = set() - for _k, a in self.items(): - for i in a.imports: - i.used.add(a) diff --git a/admin/sitegen-lib/config.py b/admin/sitegen-lib/config.py deleted file mode 100644 --- a/admin/sitegen-lib/config.py +++ /dev/null @@ -1,32 +0,0 @@ -# pattern for release tarball filename -release_pattern = r"""^afp-(.*)-([0-9\-]{10}).tar.gz$""" - -### licenses - -# key : (title, url) -# 'key' denotes the short name of the license (e. g. 'LGPL') -# 'title' denotes the display title of the license -# (e. g. 'GNU Lesser General Public License (LGPL)') -# 'url' contains the url of the license text - -licenses = { - 'LGPL': ("GNU Lesser General Public License (LGPL)", - "http://isa-afp.org/LICENSE.LGPL"), - 'BSD': ("BSD License", "http://isa-afp.org/LICENSE"), -} - - - -### options - -class Options(object): - def __init__(self): - self.dest_dir = None - self.metadata_dir = None - self.thys_dir = None - self.templates_dir = None - self.status_file = None - self.build_download = False - self.is_devel = False - -options = Options() diff --git a/admin/sitegen-lib/metadata.py b/admin/sitegen-lib/metadata.py deleted file mode 100644 --- a/admin/sitegen-lib/metadata.py +++ /dev/null @@ -1,87 +0,0 @@ -import json -import terminal -from config import licenses - -def parse_extra(extra, **kwargs): - k, v = extra.split(":", 1) - return k.strip(), v.strip() - -# extracts name and URL from 'name ' as a pair -def parse_name_url(name, entry, key): - if name.find(" and ") != -1: - terminal.warn(u"In entry {0}: {1} field contains 'and'. Use ',' to separate names.".format(entry, key)) - url_start = name.find('<') - url_end = name.find('>') - if url_start != -1 and url_end != -1: - url = name[url_start+1:url_end].strip() - if url.startswith("mailto:"): - url = url.replace("@", " /at/ ").replace(".", " /dot/ ") - elif "@" in url: - terminal.warn(u"In entry {0}: Found mail address without 'mailto:': {1}".format(entry, url)) - url = "mailto:" + url - url = url.replace("@", " /at/ ").replace(".", " /dot/ ") - return name[:url_start].strip(), url - else: - terminal.warn(u"In entry {0}: no URL specified for {1} {2} ".format(entry, key, name)) - return name, "" - -def parse_author(author, entry, key): - return parse_name_url(author, entry, key) - -def parse_contributors(contributor, entry, key): - if contributor == "": - return "", None - else: - return parse_name_url(contributor, entry, key) - -def parse_license(name, **kwargs): - if name not in licenses: - raise ValueError(u"Unknown license {0}".format(name)) - return licenses[name] - -def parse_email(email, entry, key): - stripped = email.strip() - if ' ' in stripped: - terminal.warn(u"In entry {0}: possibly malformed email in field {1}: '{2}'".format(entry, key, email)) - return stripped - -def empty_deps(entries): - d = {} - for e in entries: - d[e] = {} - d[e]["afp_deps"] = {} - d[e]["distrib_deps"] = {} - return d - -def read_deps(f): - #TODO: make code fail safe - j = json.load(f) - d = {} - for entry in j: - for e in entry: - d[e] = entry[e] - return d - -# key : (split, processor, default) -# 'key' denotes the key of the key-value pair in the metadata file -# if it ends with a '*', e. g. 'extra*' and you have two keys 'extra-foo' -# and 'extra-bar' in your metadata file, you will get: -# attributes['extra'] == { 'foo': , 'bar': } -# 'split' if False, the value will be treated as a simple string, otherwise -# it will be split at ',' -# 'processor' if defined, the callable will be invoked with each string (or -# substring if split is 'True') and the result is used -# 'default' is optional and specifies a default value, which will be treated -# as if it has been read from the file, i. e. is subject to splitting and -# processing -attribute_schema = { - 'topic': (True, None, None), - 'date': (False, None, None), - 'author': (True, parse_author, None), - 'contributors': (True, parse_contributors, ""), - 'title': (False, None, None), - 'abstract': (False, None, None), - 'license': (False, parse_license, "BSD"), - 'extra*': (False, parse_extra, None), - 'notify': (True, parse_email, "") -} diff --git a/admin/sitegen-lib/sitegen.py b/admin/sitegen-lib/sitegen.py deleted file mode 100755 --- a/admin/sitegen-lib/sitegen.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python - -## Dependencies: Python 3.5 -## -## This script reads a metadata file and generates the topics.html, -## index.html and the entry pages on isa-afp.org. -## -## For meta data documentation see `metadata/README` -## For adding new entries see `doc/editors/new-entry-checkin.html` -## - -from six.moves import configparser - -from collections import OrderedDict -import argparse -from sys import stderr -from functools import partial -from operator import itemgetter -import codecs -import os -import re -import json - -from termcolor import colored - -# modules -from config import options, release_pattern -import metadata -from terminal import warn, error -import templates -import afpstats - -# performs a 'diff' between metadata and the actual filesystem contents -def check_fs(meta_entries, directory): - def is_fs_entry(e): - root = os.path.join(directory, e) - return os.path.isdir(root) and not os.path.exists(os.path.join(root, ".sitegen-ignore")) - fs_entries = set(e for e in os.listdir(directory) if is_fs_entry(e)) - meta_entries = set(k for k, _ in meta_entries.items()) - # check for entries not existing in filesystem - for fs_missing in meta_entries - fs_entries: - print(colored(u"Check: In metadata: entry {0} doesn't exist in filesystem".format(fs_missing), - 'yellow', attrs=['bold']), file=stderr) - for meta_missing in fs_entries - meta_entries: - print(colored(u"Check: In filesystem: entry {0} doesn't exist in metadata".format(meta_missing), - 'yellow', attrs=['bold']), file=stderr) - return len(fs_entries ^ meta_entries) - -# takes the 'raw' data from metadata file as input and performs: -# * checking of data against attribute_schema -# * defaults for missing keys -# * elimination of extraneous keys -# * splitting at ',' if an array is requested -def validate(entry, attributes): - sane_attributes = {} - missing_keys = [] - processed_keys = set() - for key, (split, processor, default) in metadata.attribute_schema.items(): - if processor is None: - processor = lambda str, **kwargs: str - if key.endswith("*"): - shortkey = key[:len(key)-1] - result = OrderedDict() - process = partial(processor, entry=entry, shortkey=shortkey) - for appkey, str in attributes.items(): - if appkey.startswith(shortkey + "-"): - processed_keys.add(appkey) - app = appkey[len(shortkey) + 1:] - if not split: - result[app] = process(str.strip(), appendix=app) - else: - result[app] = [process(s.strip(), appendix=app) for s in str.split(',')] - sane_attributes[shortkey] = result - else: - process = partial(processor, entry=entry, key=key) - if default is None and key not in attributes: - missing_keys.append(key) - sane_attributes[key] = process("") if not split else [] - else: - value = attributes[key] if key in attributes else default - processed_keys.add(key) - if not split: - sane_attributes[key] = process(value) - else: - sane_attributes[key] = [process(s.strip()) for s in value.split(',')] - if missing_keys: - error(u"In entry {0}: missing key(s) {1!s}".format(entry, missing_keys), abort = True) - - extraneous_keys = set(attributes.keys()) - processed_keys - if extraneous_keys: - warn(u"In entry {0}: unknown key(s) {1!s}. Have you misspelled them?".format(entry, list(extraneous_keys))) - - return sane_attributes - -# reads the metadata file and returns a dict mapping each entry to the attributes -# specified. one can rely upon that they conform to the attribute_schema -def parse(filename): - parser = configparser.RawConfigParser(dict_type=OrderedDict) - try: - parser.read_file(codecs.open(filename, encoding='UTF-8', errors='strict')) - return OrderedDict((sec, validate(sec, dict(parser.items(sec)))) - for sec in parser.sections()) - except UnicodeDecodeError as ex: - error(u"In file {0}: invalid UTF-8 character".format(filename), exception=ex, abort=True) - -# reads the version file, composed of pairs of version number and release date -def read_versions(filename): - versions = [] - try: - with open(filename) as input: - for line in input: - try: - version, release_date = line.split(" = ") - except ValueError as ex: - error(u"In file {0}: Malformed association {1}".format(filename, line), exception=ex) - error("Not processing releases") - return [] - else: - versions.append((version, release_date.strip())) - except Exception as ex: - error(u"In file {0}: error".format(filename), exception=ex) - error("Not processing releases") - return [] - else: - versions.sort(key=itemgetter(1), reverse=True) - return versions - -# reads the list of entry releases (metadata/releases) -def associate_releases(entries, versions, filename): - for _, attributes in entries.items(): - attributes['releases'] = OrderedDict() - prog = re.compile(release_pattern) - warnings = {} - try: - with open(filename) as input: - lines = [] - for line in input: - line = line.strip() - result = prog.match(line) - try: - entry, date = result.groups() - except ValueError as ex: - error(u"In file {0}: Malformed release {1}".format(filename, line.replace), exception=ex) - else: - if not entry in entries: - if not entry in warnings: - warnings[entry] = [line] - else: - warnings[entry].append(line) - else: - lines.append((entry, date)) - for entry, releases in warnings.items(): - warn(u"In file {0}: In release(s) {1!s}: Unknown entry {2}".format(filename, releases, entry)) - lines.sort(reverse=True) - for line in lines: - found = False - entry, date = line - for version_number, version_date in versions: - if version_date <= date: - entry_releases = entries[entry]['releases'] - if version_number not in entry_releases: - entry_releases[version_number] = [] - entry_releases[version_number].append(date) - found = True - break - if not found: - warn(u"In file {0}: In release {1}: Release date {2} has no matching version".format(filename, line, date)) - except Exception as ex: - error(u"In file {0}: error".format(filename), exception=ex) - error("Not processing releases") - -def parse_status(filename): - with open(filename) as input: - data = json.load(input) - - build_data = data['build_data'] - status = dict() - for entry in data['entries']: - status[entry['entry']] = entry['status'] - - return build_data, status - -def main(): - usage = "sitegen.py [-h] [--templates TEMPLATES_DIR --dest DEST_DIR] [--status STATUS_FILE] [--deps DEPS_FILE] METADATA_DIR THYS_DIR" - parser = argparse.ArgumentParser(usage=usage) - parser.add_argument("metadata_dir", metavar="METADATA_DIR", action="store", - help="metadata location") - parser.add_argument("thys_dir", metavar="THYS_DIR", action="store", - help="directory with afp entries") - parser.add_argument("--templates", action="store", dest="templates_dir", - help="directory with Jinja2 templates") - parser.add_argument("--dest", action="store", dest="dest_dir", - help="destination dir for generated html files") - parser.add_argument("--status", action="store", dest="status_file", - help="status file location (devel)") - parser.add_argument("--deps", action="store", dest="deps_file", - help="dependencies file location") - - parser.parse_args(namespace=options) - options.is_devel = options.status_file is not None - - if options.dest_dir and not options.templates_dir: - error("Please specify templates dir", abort=True) - - # parse metadata - entries = parse(os.path.join(options.metadata_dir, "metadata")) - versions = read_versions(os.path.join(options.metadata_dir, "release-dates")) - associate_releases(entries, versions, os.path.join(options.metadata_dir, "releases")) - if len(entries) == 0: - warn("In metadata: No entries found") - - # generate depends-on, used-by entries, lines of code and number of lemmas - # by using an afp_dict object - # TODO: error instead of warn - deps_dict = metadata.empty_deps(entries) - if options.deps_file: - with open(options.deps_file, 'r') as df: - deps_dict = metadata.read_deps(df) - else: - warn("No dependencies file specified") - - afp_dict = afpstats.afp_dict(entries, options.thys_dir, deps_dict) - afp_dict.build_stats() - for e in entries: - entries[e]['depends-on'] = list(map(str, afp_dict[e].imports)) - entries[e]['used-by'] = list(map(str, afp_dict[e].used)) - - # perform check - count = check_fs(entries, options.thys_dir) - output = "Checked directory {0}. Found {1} warnings.".format(options.thys_dir, count) - color = 'yellow' if count > 0 else 'green' - print(colored(output, color, attrs=['bold'])) - - # perform generation - if options.dest_dir: - if options.status_file is not None: - (build_data, status) = parse_status(options.status_file) - for a in afp_dict: - if a in status: - afp_dict[a].status = status[a] - else: - afp_dict[a].status = "skipped" - else: - build_data = dict() - - builder = templates.Builder(options, entries, afp_dict) - builder.generate_topics() - builder.generate_index() - builder.generate_entries() - builder.generate_statistics() - builder.generate_download() - for s in ["about", "citing", "updating", "search", "submitting", - "using"]: - builder.generate_standard(s + ".html", s + ".tpl") - builder.generate_rss(30) - #TODO: look over it one more time - if options.is_devel: - builder.generate_status(build_data) - -if __name__ == "__main__": - main() diff --git a/admin/sitegen-lib/statistics.py b/admin/sitegen-lib/statistics.py deleted file mode 100644 --- a/admin/sitegen-lib/statistics.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Most the statistics for the site, are generated by Hugo. This script, -generates other statistics like number of lines in the AFP using the -scripts from the current AFP. - -For this script to work, `return data` needs to be added at -line 212 in templates.py -""" - -import os - -import afpstats -import metadata -import templates -from config import options -from sitegen import associate_releases, parse, read_versions -from write_file import write_file - - -def add_statistics(base_dir, thys_dir, data_dir): - """Creates the necessary objects to generates the statistics, - then outputs them to the data directory""" - options.templates_dir = os.path.join(base_dir, "metadata", "templates") - options.dest_dir = data_dir - - entries = parse(os.path.join(base_dir, "metadata", "metadata")) - versions = read_versions(os.path.join(base_dir, "metadata", "release-dates")) - associate_releases(entries, versions, os.path.join(base_dir, "metadata", "releases")) - - deps_dict = metadata.empty_deps(entries) - - afp_dict = afpstats.afp_dict(entries, thys_dir, deps_dict) - afp_dict.build_stats() - builder = templates.Builder(options, entries, afp_dict) - - stats = builder.generate_statistics() - - loc_articles = [article.loc for article in stats["articles_by_time"]] - - all_articles = [a.name for a in stats["articles_by_time"]] - - data = { - "num_lemmas": stats["num_lemmas"], - "num_loc": stats["num_loc"], - "articles_year": stats["articles_year"], - "loc_years": stats["loc_years"], - "author_years": stats["author_years"], - "author_years_cumulative": stats["author_years_cumulative"], - "loc_articles": loc_articles, - "all_articles": all_articles, - } - - write_file(os.path.join(data_dir, "statistics.json"), data) diff --git a/admin/sitegen-lib/templates.py b/admin/sitegen-lib/templates.py deleted file mode 100644 --- a/admin/sitegen-lib/templates.py +++ /dev/null @@ -1,231 +0,0 @@ -from collections import OrderedDict -from itertools import groupby -import os -import datetime -from jinja2 import Environment, FileSystemLoader - -import terminal - -### topics - -class Tree(object): - def __init__(self): - self.subtopics = OrderedDict() - self.entries = [] - - def add_topic(self, topic): - if len(topic) > 0: - if topic[0] not in self.subtopics: - tree = Tree() - self.subtopics[topic[0]] = tree - else: - tree = self.subtopics[topic[0]] - tree.add_topic(topic[1:]) - - def add_to_topic(self, topic, entry): - if len(topic) > 0: - if topic[0] not in self.subtopics: - terminal.error(u"In entry {0}: unknown (sub)topic {1}".format(entry, topic), abort=True) - else: - self.subtopics[topic[0]].add_to_topic(topic[1:], entry) - else: - self.entries.append(entry) - - def __str__(self): - return self._to_str() - - def _to_str(self, indent=0): - indent_str = ' ' * indent - result = indent_str + str(self.entries) + "\n" - for subtopic, tree in self.subtopics.items(): - result += indent_str - result += subtopic - result += "\n" - result += tree._to_str(indent + 2) - return result - -def read_topics(filename): - tree = Tree() - stack = [] - with open(filename) as f: - for line in f: - count = 0 - while line[count] == ' ': - count += 1 - if count % 2: - raise Exception(u"Illegal indentation at line '{0}'".format(line)) - level = count // 2 - if level <= len(stack): - stack = stack[0:level] - else: - raise Exception(u"Illegal indentation at line '{0}'".format(line)) - stack.append(line[count:len(line)-1]) - tree.add_topic(stack) - return tree - -# for topics page: group entries by topic -def collect_topics(entries, metadata_dir): - tree = read_topics(os.path.join(metadata_dir, "topics")) - for entry, attributes in entries.items(): - for topic in attributes['topic']: - tree.add_to_topic([s.strip() for s in topic.split('/')], entry) - return tree - - -class Builder(): - """Contains environment for building webpages from templates""" - - def __init__(self, options, entries, afp_entries): - self.j2_env = Environment(loader=FileSystemLoader(options.templates_dir), - trim_blocks=True) - # pass functions to environment for use in templates - self.prepare_env() - self.options = options - #TODO: use only afp_entries - self.entries = entries - self.afp_entries = afp_entries - - def prepare_env(self): - def startswith(value, beginning): - return value.startswith(beginning) - def datetimeformat(value, format_str='%Y-%m-%d'): - return value.strftime(format_str) - def rfc822(value): - # Locale could be something different than english, to prevent printing - # non english months, we use this fix - month = "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split(" ")[value.month - 1] - return value.strftime("%d " + month + " %Y %T %z") - def split(value): - return value.split() - def short_month(value): - return "jan feb mar apr may jun jul aug sep oct nov dec".split(" ")[value - 1] - self.j2_env.filters['startswith'] = startswith - self.j2_env.filters['datetimeformat'] = datetimeformat - self.j2_env.filters['rfc822'] = rfc822 - self.j2_env.filters['split'] = split - self.j2_env.filters['short_month'] = short_month - - def write_file(self, filename, template, values): - # UTF-8 hack because of different string handling in python 2 vs 3 - with open(os.path.join(self.options.dest_dir, filename), 'wb') as f: - f.write(template.render(values).encode('utf8')) - - def generate_standard(self, filename, template_name): - template = self.j2_env.get_template(template_name) - self.write_file(filename, template, {}) - terminal.success("Generated {}".format(filename)) - - def generate_topics(self): - tree = collect_topics(self.entries, self.options.metadata_dir) - template = self.j2_env.get_template("topics.tpl") - self.write_file("topics.html", template, {'tree': tree}) - terminal.success("Generated topics.html") - - def generate_index(self): - data = {'is_devel': self.options.is_devel} - by_year = groupby(sorted(self.afp_entries.values(), - key=lambda e: (e.publish_date, e.name), - reverse=True), - key=lambda e: e.publish_date.year) - data['by_year'] = [(year, list(entries)) for year, entries in by_year] - template = self.j2_env.get_template("index.tpl") - self.write_file("index.html", template, data) - terminal.success("Generated index.html") - - def generate_entries(self): - counter = 0 - template = self.j2_env.get_template("entry.tpl") - for name, entry in self.afp_entries.items(): - self.write_file(os.path.join("entries", name + ".html"), template, - {'entry': entry, 'is_devel': self.options.is_devel, - 'ROOT_PATH': '../'}) - counter += 1 - for name, entry in self.afp_entries.no_index.items(): - self.write_file(os.path.join("entries", name + ".html"), template, - {'entry': entry, 'is_devel': self.options.is_devel, - 'ROOT_PATH': '../'}) - counter += 1 - terminal.success("Generated html files for {:d} entries".format(counter)) - - def generate_download(self): - template = self.j2_env.get_template("download.tpl") - self.write_file("download.html", template, - {'is_devel': self.options.is_devel}) - terminal.success("Generated download.html") - - def generate_statistics(self): - #TODO: simplify with itertools - # Count loc and articles per year - articles_years = dict() - loc_years = dict() - for article in self.afp_entries.values(): - try: - articles_years[article.publish_date.year] += 1 - loc_years[article.publish_date.year] += article.loc - except KeyError: - articles_years[article.publish_date.year] = 1 - loc_years[article.publish_date.year] = article.loc - # Count new authors per year - author_years = dict.fromkeys(articles_years.keys(), 0) - for author in self.afp_entries.authors.values(): - first_year = min([e.publish_date.year for e in author.articles]) - try: - author_years[first_year] += 1 - except KeyError: - author_years[first_year] = 1 - # Build cumulative values - author_years_cumulative = author_years.copy() - for y in sorted(articles_years)[1:]: - articles_years[y] += articles_years[y - 1] - loc_years[y] += loc_years[y - 1] - author_years_cumulative[y] += author_years_cumulative[y - 1] - data = {'entries': self.afp_entries} - data['num_lemmas'] = sum([a.lemmas for a in self.afp_entries.values()]) - data['num_loc'] = sum([a.loc for a in self.afp_entries.values()]) - data['years'] = sorted(articles_years) - data['articles_year'] = [articles_years[y] for y - in sorted(articles_years)] - data['loc_years'] = [round(loc_years[y], -2) for y in sorted(loc_years)] - data['author_years'] = [author_years[y] for y in sorted(author_years)] - data['author_years_cumulative'] = [author_years_cumulative[y] for y in - sorted(author_years_cumulative)] - # Find 10 most imported entries, entries with the same number of - # imports share one place. - most_used = sorted([a for a in self.afp_entries.values()], - key=lambda x: (-len(x.used), x.name)) - # Show more than 10 articles but not more than necessary - i = 0 - while (i < 10 or (i + 1 < len(most_used) and - len(most_used[i].used) == len(most_used[i + 1].used))): - i += 1 - # Groupby iterators trigger some obscure bug in jinja2 - # https://github.com/pallets/jinja/issues/555 - # So don't use groupby iterator directly and convert to list of lists - data['most_used'] = [(len_used, list(articles)) for (len_used, articles) - in groupby(most_used[:i + 1], - key=lambda x: len(x.used))] - data['articles_by_time'] = sorted(self.afp_entries.values(), - key=lambda x: x.publish_date) - data['articles_per_year'] = [(year, list(articles)) for (year, articles) - in groupby(data['articles_by_time'], - key=lambda x: x.publish_date.year)] - template = self.j2_env.get_template("statistics.tpl") - self.write_file("statistics.html", template, data) - terminal.success("Generated statistics.html") - return data - - def generate_status(self, build_data): - template = self.j2_env.get_template("status.tpl") - self.write_file("status.html", template, - {'entries': [self.afp_entries[e] for e - in sorted(self.afp_entries)], - 'build_data': build_data}) - terminal.success("Generated status.html") - - def generate_rss(self, num_entries): - entries = sorted(self.afp_entries.values(), - key=lambda e: (e.publish_date, e.name), - reverse=True) - template = self.j2_env.get_template("rss.tpl") - self.write_file("rss.xml", template, {'entries': entries[:num_entries]}) - terminal.success("Generated rss.xml") diff --git a/admin/sitegen-lib/terminal.py b/admin/sitegen-lib/terminal.py deleted file mode 100644 --- a/admin/sitegen-lib/terminal.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import print_function -from sys import stderr -from termcolor import colored - -from config import options - -def warn(message): - print(colored(u"Warning: {0}".format(message), 'yellow', attrs=['bold']), file=stderr) - -def error(message, exception=None, abort=False): - print(colored(u"Error: {0}".format(message), 'red', attrs=['bold']), file=stderr) - if exception: - error("*** exception message:") - error(u"*** {0!s} {1!s}".format(exception, type(exception))) - if abort: - error("Fatal. Aborting") - exit(1) - -def success(message): - print(colored("Success: {0}".format(message), 'green'), file=stderr) diff --git a/admin/sitegen-lib/write_file.py b/admin/sitegen-lib/write_file.py deleted file mode 100644 --- a/admin/sitegen-lib/write_file.py +++ /dev/null @@ -1,18 +0,0 @@ -import json -import os - - -def write_file(file, data, write=True, overwrite=False): - file_exists = os.path.isfile(file) - - if file_exists and not overwrite: - with open(file) as r: - original_data = json.load(r) - - data = {**original_data, **data} - - # Only write file if write or if file doesn't exist - # Or, only don't write if write is false and file exists - if not file_exists or write: - with open(file, "w", encoding="utf-8") as w: - json.dump(data, w, ensure_ascii=False, indent=4)