Lexesis/run_tests.py

#!/usr/bin/python3
# Lexesis - A language agnostic lexical analyser generator
# Copyright © 2016-2017 Thomas Avé, Robin Jadoul
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import unittest, subprocess, filecmp, argparse, os.path, os, shutil

REFERENCE_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "tests")

REGEXES_DATA = [    # (regexes, should be accepted)
        ([r"[a-zA-Z0-9]*"], True),
        ([r"[a-dD-F9-0]*"], True),
        ([r"a", r"a|b"], True),
        ([r"a|b", r"a"], True),

        ([r"[[]", r"[]]", r"[]-[]", r"[][-]"], True),
        ([r"[^^]", r"[^]-[]", r"[^][-]"], True),
        ([r"[ ^]", r"[ ^-X ]", r"[]-^]"], True),
        ([r"[^-X]", r"[^][]"], True),
        ([r"[-]", r"[]-]"], True),
        ([r"[^-]", "[^]]", r"[^-]", r"[^-^]"], True),
        ([r"[^--]", r"[--]"], True),
        ([r"[^]"], False),
        ([r"[]"], False),

        ([r"\r", r"\n", r"\f", r"\v", r"\a", r"\t", r"\b", r"\s", r" "], True),
        ([r"\\", r"\*", r"\+", r"\|", r"\(", r"\)", r"\[", r"\[", r"\?", r"\."], True),

        ([r".", r"a*", r"a+", r"a?", r"(ab)*", r"a|b"], True),
        ([r"(a|b)*", r"(a|b)+", r"a**", r"a??", r"(ab)?"], True),
        ([r"("], False),
        ([r")"], False),
        ([r"["], False),
        ([r"]"], False),
        ([r"(test(a)qwerty"], False),
        ([r"test(a)qwerty)"], False),
        ([r"ab|"], False),
        ([r"|ab"], False),
        ([r"ab(c|)de"], False),
        ([r"ab(|c)de"], False),

        ([r""], False),
        ([r"()"], False),
        ([r"(?)"], False),
        ([r"((ab)c()de)"], False),
        ([r"|"], False),
        ([r"(|)"], False),
        ([r"?"], False),
        ([r"*"], False),
        ([r"(*)"], False),
        ([r"+"], False),
        ([r"(+)"], False),
        ]

EXAMPLE_TESTS = {   #Mapping from test name to executable and number of available test input files
        "keywords" : ("examples/keywords/keywords", 2),
        "highlighter": ("examples/SyntaxHighlighter/highlighter", 8),
        "leopard": ("examples/leopard/leopard", 8),
        }

def make_pipeline_test(regexes, should_accept, idx):
    def test(self):
        p = subprocess.Popen([os.path.join(args.builddir, "src", "Lexesis-test"), "test_%s" % idx, data_dir], stdin=subprocess.PIPE, stderr=subprocess.DEVNULL)
        p.communicate(bytes("\n".join(regexes) + "\n", "utf-8"))
    
        if should_accept:
            self.assertEqual(0, p.returncode)

            self.assertTrue(filecmp.cmp(os.path.join(REFERENCE_DIR, "test_%s.re" % idx), os.path.join(data_dir, "test_%s.re" % idx)), "The regex for testcase %s is incorrect" % idx)
            self.assertTrue(filecmp.cmp(os.path.join(REFERENCE_DIR, "test_%s.enfa" % idx), os.path.join(data_dir, "test_%s.enfa" % idx)), "The eNFA for testcase %s is incorrect" % idx)
            self.assertTrue(filecmp.cmp(os.path.join(REFERENCE_DIR, "test_%s.mssc" % idx), os.path.join(data_dir, "test_%s.mssc" % idx)), "The mssc'd DFA for testcase %s is incorrect" % idx)
            self.assertTrue(filecmp.cmp(os.path.join(REFERENCE_DIR, "test_%s.min" % idx), os.path.join(data_dir, "test_%s.min" % idx)), "The minimized DFA for testcase %s is incorrect" % idx)
        else:
            self.assertNotEqual(0, p.returncode)

    return test

def make_functional_test(name, prog, idx):
    def test(self):
        inpath = os.path.join(REFERENCE_DIR, "%s_%s.in" % (name, idx))
        infile = open(inpath)
        outpath = os.path.join(data_dir, "%s_%s.out" % (name, idx))
        outfile = open(outpath, "w")
        checkpath = os.path.join(REFERENCE_DIR, "%s_%s.exp" % (name, idx))

        p = subprocess.Popen([prog], stdin=infile, stdout=outfile)
        p.communicate()

        infile.close()
        outfile.close()

        self.assertTrue(filecmp.cmp(checkpath, outpath), "Testcase %s for example program %s failed" % (idx, name))

    return test

class Tests(unittest.TestCase):
    pass

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--builddir", metavar="builddir", default=os.path.join(os.path.abspath(os.path.dirname(__file__)), "build"), required=False)
    args = parser.parse_args()

    data_dir = os.path.join(args.builddir, "test_data")
    try:
        shutil.rmtree(data_dir)
    except Exception as err:
        pass
    os.mkdir(data_dir)

    for i, (regexes, should_accept) in enumerate(REGEXES_DATA):
        t = make_pipeline_test(regexes, should_accept, i)
        setattr(Tests, "test_%s" % i, t)

    for test, (prog, num) in EXAMPLE_TESTS.items():
        for i in range(num):
            t = make_functional_test(test, os.path.join(args.builddir, prog), i)
            setattr(Tests, "test_%s_%s" % (test, i), t)

    unittest.main()