Solution for line-by-line .MC "conditional compilation"

Interesting solution to the problem!

I had to solve a similar problem back in 2015 / 2016 for Note2Watch with the main challenge being how to build a widget and watch-app from the same code base. Lots of little things had to change all over the place and I'm not sure annotations existed at the time, otherwise I think I might have gone with something more like your approach.

Instead I went with the most brain-dead, cave-man approach I could think of: programmatically commenting out the code in-place that I wasn't using.

The way this worked is you wrapped certain lines in C style #ifdef / #endif lines (slightly different syntax) and then a Python script would parse the source files and perform the necessary comment/uncomment fix-ups. Then all I had to do was use Makefile targets to execute this fix-up script so I could write something like:

make widget

make watch-app

Eventually I even add different development modes I would swap in, so that added another degree of freedom, e.g.:

make staging-widget

make release-watch-app

Luckily for Runcasts, I don't need nearly this level of flexibility since I'm only targeting an audio app. To implement different release environments like testing, staging, and production, I programmatically generate a strings XML resource and then reference that in the code using Ui.loadResource(). Then I use a JSON file as the source of truth, an XML template to inject the values from JSON, a bash script to perform the interpolation using sed, and Makefile targets to hide the (slight bit of) messiness. So far that pattern has worked really well, and it's much, much less code.

If anyones curious what the monkeyC metalanguage parse looked like, here's the code:

#!/usr/bin/env python
"""
Perform preprocessing on MonkeyC files.

For example allows you to write code like:

    //#if DRY_RUN
    //%    Logging.log("Would have done foo");
    //#else
        doFoo();
    //#endif

If -t DRY_RUN is passed to the preprocessor, then the `Logging.log` line will
be uncommented and the `doFoo` line will be commented out.

Usage:
    preprocessor -t <tag1> -t <tag2>


NOTE:
    This is experimental, it could absolutely destroy all of your work!
    Make sure you have your code checked-in/backed-up before running this on
    it!
"""
import argparse
import collections
import fnmatch
import os
import subprocess
import sys

COMMENT_MARKER = '//%'
COMMAND_MARKER = '//#'

VERBOSE = False
STATS = collections.Counter()


def log(msg):
    if VERBOSE:
        print >> sys.stderr, msg



class TagException(Exception):
    def __init__(self, tag, line_number):
        self.tag = tag
        self.line_number = line_number


class IfWithoutEnd(TagException):
    pass


class ElseWithoutIf(TagException):
    pass


class ElifWithoutIf(TagException):
    pass


class EndIfWithoutIf(TagException):
    pass


class NestedIfsNotAllowed(TagException):
    pass


class NestedElsesNotAllowed(TagException):
    pass


class PathNotFound(Exception):
    def __init__(self, path):
        self.path = path


class UnknownTransform(TagException):
    def __init__(self, transform, line_number):
        super(UnknownTransform, self).__init__(None, line_number)
        self.transform = transform


class UnknownCommand(TagException):
    def __init__(self, command, line_number):
        super(UnknownCommand, self).__init__(None, line_number)
        self.command = command


def diff(file1, file2):
    try:
        subprocess.check_output(['which', 'colordiff'])
    except subprocess.CalledProcessError:
        # No colordiff available
        subprocess.call(['diff', '-u', file1, file2])
    else:
        proc = subprocess.Popen(['diff', '-u', file1, file2],
                                stdout=subprocess.PIPE)
        subprocess.call(['colordiff'], stdin=proc.stdout)
        proc.wait()


def comment_line(tag, line_number, line):
    touched = False
    if not line.startswith(COMMENT_MARKER):
        line = COMMENT_MARKER + line
        touched = True
    return line, touched


def uncomment_line(tag, line_number, line):
    touched = False
    if line.startswith(COMMENT_MARKER):
        line = line[len(COMMENT_MARKER):]
        touched = True
    return line, touched


def transform_file(infile_path, dry_run=False, tags=None):
    tags = set(tags or [])
    cur_cmd = None
    cur_tag = None
    aux_tags = set()
    matches = set()
    begin_line_number = None

    if not os.path.exists(infile_path):
        raise PathNotFound(infile_path)

    outfile_path = infile_path + '.tmp'
    outfile = open(outfile_path, 'w')
    try:
        with open(infile_path) as infile:
            for line_number, line in enumerate(infile, start=1):
                if line.startswith(COMMAND_MARKER):
                    parts = line.replace(COMMAND_MARKER, '').strip().split()
                    cmd = parts[0]
                    if cmd == 'if':
                        if cur_cmd == 'if':
                            raise NestedIfsNotAllowed(cur_tag, line_number)
                        tag = parts[1]
                        log( "{}:{}: if '{}' found".format(
                                infile_path, line_number, tag))
                        cur_cmd = cmd
                        cur_tag = tag
                        begin_line_number = line_number
                    elif cmd == 'else':
                        if cur_cmd == 'else':
                            raise NestedElsesNotAllowed(cur_tag, line_number)
                        elif cur_cmd not in ('if', 'elif'):
                            raise ElseWithoutIf(cur_tag, line_number)
                        log( "{}:{}: else '{}' found".format(
                                infile_path, line_number, cur_tag))
                        cur_cmd = cmd
                        begin_line_number = line_number
                    elif cmd == 'elif':
                        if cur_cmd not in ('if', 'elif'):
                            raise ElifWithoutIf(cur_tag, line_number)
                        tag = parts[1]
                        log( "{}:{}: elif '{}' found".format(
                                infile_path, line_number, tag))
                        aux_tags.add(cur_tag)
                        cur_cmd = cmd
                        cur_tag = tag
                        begin_line_number = line_number
                    elif cmd == 'endif':
                        log( "{}:{}: endif '{}' found".format(
                                infile_path, line_number, cur_tag))
                        if cur_cmd not in ('if', 'else'):
                            raise EndIfWithoutIf(tag, line_number)
                        cur_cmd = None
                        cur_tag = None
                        aux_tags = set()
                        matches = set()
                        begin_line_number = None
                    else:
                        raise UnknownCommand(cmd, line_number)
                elif cur_cmd == 'if':
                    if cur_tag in tags:
                        transform_func = uncomment_line
                        matches.add(cur_tag)
                    else:
                        transform_func = comment_line
                    line, touched = transform_func(tag, line_number, line)
                    if touched:
                        STATS[infile_path] += 1
                elif cur_cmd == 'elif':
                    if matches and cur_tag not in matches:
                        transform_func = comment_line
                    elif cur_tag in tags:
                        transform_func = uncomment_line
                        matches.add(cur_tag)
                    else:
                        transform_func = comment_line
                    line, touched = transform_func(tag, line_number, line)
                    if touched:
                        STATS[infile_path] += 1
                elif cur_cmd == 'else':
                    if (aux_tags | {cur_tag}) & tags:
                        transform_func = comment_line
                    else:
                        transform_func = uncomment_line
                    line, touched = transform_func(tag, line_number, line)
                    if touched:
                        STATS[infile_path] += 1

                # No matter what, write the output line
                outfile.write(line)
    except:
        # Don't leave *.tmp files laying around
        outfile.close()
        os.unlink(outfile_path)
        raise
    else:
        outfile.close()

    if cur_cmd:
        os.unlink(outfile_path)
        if cur_cmd == 'if':
            raise IfWithoutEnd(cur_tag, begin_line_number)
        else:
            assert False

    if VERBOSE:
        diff(infile_path, outfile_path)
    if dry_run:
        os.unlink(outfile_path)
    else:
        os.rename(outfile_path, infile_path)


def do_transform_file(args, infile_path):
    try:
        transform_file(infile_path, dry_run=args.dry_run, tags=args.tags)
    except ElifWithoutIf as e:
        sys.exit("error: {}:{}: elif '{}' without beginning if".format(
            infile_path, e.line_number, e.tag))
    except ElseWithoutIf as e:
        sys.exit("error: {}:{}: else '{}' without beginning if".format(
            infile_path, e.line_number, e.tag))
    except EndIfWithoutIf as e:
        sys.exit("error: {}:{}: endif '{}' without beginning if".format(
            infile_path, e.line_number, e.tag))
    except IfWithoutEnd as e:
        sys.exit("error: {}:{}: if '{}' without accompanying endif".format(
            infile_path, e.line_number, e.tag))
    except NestedIfsNotAllowed as e:
        sys.exit("error: {}:{}: tag '{}' already active, nested-ifs not"
                 " allowed".format(infile_path, e.line_number, e.tag))
    except NestedElsesNotAllowed as e:
        sys.exit("error: {}:{}: tag '{}' already active, nested-elses not"
                 " allowed".format(infile_path, e.line_number, e.tag))
    except UnknownTransform as e:
        sys.exit("error: {}:{}: unknown transform '{}'".format(
            infile_path, e.line_number, e.transform))
    except UnknownCommand as e:
        sys.exit("error: {}:{}: unknown command '{}'".format(
            infile_path, e.line_number, e.command))
    except PathNotFound as e:
        sys.exit("error: '{}' not found".format(e.path))


def do_collect_tag_from_file(args, infile_path, collected_tags):
    with open(infile_path) as infile:
        for line in infile:
            if line.startswith(COMMAND_MARKER):
                parts = line.replace(COMMAND_MARKER, '').strip().split()
                cmd = parts[0]
                if cmd == 'if':
                    tag = parts[1]
                    collected_tags.add(tag)


def do_yield_files(args):
    for path in args.paths:
        if not os.path.exists(path):
            sys.exit("error: '{}' not found".format(path))
        if os.path.isdir(path):
            for root, dirs, files in os.walk(path):
                for filename in files:
                    filepath = os.path.join(root, filename)
                    if args.globs:
                        for glob in args.globs:
                            if fnmatch.fnmatch(filepath, glob):
                                break
                        else:
                            continue
                    yield filepath
        elif os.path.isfile(path):
            yield path
        else:
            sys.exit("error: what kind of file is '{}'?".format(path))


def main():
    global VERBOSE
    parser = argparse.ArgumentParser(
            description="Perform preprocessing on source files")
    parser.add_argument('paths', nargs='+',
                        help='Files or directories to operate on')
    parser.add_argument('--dry-run', action='store_true',
                        help="Don't write results")
    parser.add_argument('-g', '--glob',
                        dest='globs',
                        default=['*.mc'],
                        action='append',
                        help="Filename filter (default: *.mc)")
    parser.add_argument('--show-tags', action='store_true',
                        help='Just show available tags')
    parser.add_argument('-t', '--tag', dest='tags', action='append',
                        default=[],
                        help='Tags to operate on (default: all tags)')
    parser.add_argument('--verbose', action='store_true', help='Verbose mode')

    args = parser.parse_args()

    VERBOSE = args.verbose

    if args.show_tags:
        # Show tags
        collected_tags = set()
        for path in do_yield_files(args):
            do_collect_tag_from_file(args, path, collected_tags)
        for tag in sorted(collected_tags):
            print tag
    else:
        # Transform
        for path in do_yield_files(args):
            do_transform_file(args, path)

        if args.verbose:
            print

        print "Filename".ljust(78 - 15 - 1), 'Lines Touched'.rjust(15)
        print "=" * 78
        total = 0
        for filename, strip_count in STATS.iteritems():
            total += strip_count
            print filename.ljust(78 - 15 - 1), str(strip_count).rjust(15)
        print "-" * 78
        print 'Total'.ljust(78 - 15 - 1), str(total).rjust(15)

        if args.dry_run:
            print "Dry-run complete"



if __name__ == '__main__':
    main()