#!/usr/bin/env python3
#
# replacement for the ./run script
# this one handle more options, use the -h 
# works also with python3
#
# when using --xparam the scrip search for line like
#   # XPARAM OPERATION=(append|1:recycle|10:generate)
# in the source and will run the scriipt with the environement variable
# OPERATION set to these different value append, recycle and generate
# the number 1 and 10 are the level at witch the value will be considered
# the level can be set using option "--xparam-level LEVEL" the default level is 9
# when no level are specified next to the value, 0 is used
#   # XPARAM OPERATION=(append|1:recycle|10:generate)
# is equivalent to
#   # XPARAM OPERATION=(0:append|1:recycle|10:generate)
# when multiple XPARAM are defined, arun handle all the possibilities
#

from __future__ import print_function

import os
import sys
import subprocess
import argparse
import threading 
import time
import re
import signal
import codecs

try:
    import configparser
except ImportError:
    # python 2
    import ConfigParser as configparser

try:
    import queue
except ImportError:
    # python 2
    import Queue as queue

class StopOnFirstError(RuntimeError):
    pass

re_first_sep=re.compile(r'\s*(?P<key>[a-zA-Z]\w*)\s*(?P<sep>=|\s)(?P<remain>.*)')
re_quote_protected=re.compile(r'"(?P<value>[^"\\]*(\\.[^"\\]*)*)"(?P<remain>.*)', re.DOTALL)
re_singlequote_protected=re.compile(r"'(?P<value>[^'\\]*(\\.[^'\\]*)*)'(?P<remain>.*)", re.DOTALL)
re_test_param=re.compile(r'[#]\s*PARAM\s+(?P<param>[a-zA-Z]\w*)\s*=\s*(?P<value>.*)')
re_test_xparam=re.compile(r'[#]\s*XPARAM\s+(?P<xparam>[a-zA-Z]\w*)\s*=\s*(?P<values>.*)')
re_test_xparam_value=re.compile(r'((?P<level>[0-9]+):)?(?P<value>.*)')
verbose=0


Status={-15:'term', -9:'kill', 0:'success', 1:'failed', -1001:'skip', -1002:'abort', -1003:'timeout' }

for k in list(Status.keys()):
    Status[Status[k]]=k

def mixer(xparams, keys=None):
    """magic function that generate all possible XPARAM scenari"""
    if not xparams:
        return
    if keys==None:
        keys=iter(xparams.keys())
    k=next(keys, None)
    if k==None:
        yield dict()
    else:
        for v in xparams[k]:
            for c in mixer(xparams, keys):
                c[k]=v
                yield c

class NonBlockingReader:

    Timeout=queue.Empty
    
    def __init__(self, stream):

        self.stream=stream
        self.queue=queue.Queue()

        def populateQueue(stream, queue):
            while True:
                line=stream.readline()
                if line:
                    queue.put(line)
                else:
                    queue.put(None)
                    break

        self.thread=threading.Thread(target=populateQueue, args=(self.stream, self.queue))
        self.thread.daemon=True
        self.thread.start()

    def readline(self, timeout = None):
        return self.queue.get(block=timeout is not None, timeout=timeout)


def parse_arg_string(st):
    params=dict()
    st=st.lstrip()
    while st:
        try:
            match=re_first_sep.match(st)
            key, sep, remain=match.group('key', 'sep', 'remain')
            if sep in [ '', ' ']:
                value=None
            elif sep=='=':
                if remain.startswith('"'):
                    match=re_quote_protected.match(remain)
                    value, remain=match.group('value', 'remain')
                elif remain.startswith("'"):
                    match=re_singlequote_protected.match(remain)
                    value, remain=match.group('value', 'remain')
                else:
                    value, remain=remain.split(None, 1)
            params[key]=value
            st=remain
        except AttributeError:
            # a failed match
            return None
    return params                   

def parse_shell_string(st):
    vars=dict()
    while st:
        var, st=st.split('=', 1)
        data=''
        while st and not st.startswith('\n'):
            if st.startswith('"'):
                match=re_quote_protected.match(st)
                value, st=match.group('value', 'remain')
            elif st.startswith("'"):
                match=re_singlequote_protected.match(st)
                value, st=match.group('value', 'remain')
            else:
                value, st=st.split(None, 1)
            data+=value
            
        vars[var]=data
        if st.startswith('\n'):
            st=st.lstrip('\n')
        
    return vars

def show_resume(output=None):
    failure=success=0
    longuest_f=longuest_s=0
    for test, label, prevail, option, returncode, duration in reports:
        if returncode!=0:
            failure+=1
            status=Status.get(returncode, 'failed')
            if not (args.dedup=='all' or args.dedup_cache=='all'):
                label=''
            outline='%-8s %3ds %s %s' % (status, duration, label, test)
            print(outline)
            if output:
                print(outline, file=output)
            longuest_f=max(longuest_f, duration)
        else:
            success+=1
            longuest_s=max(longuest_s, duration)
    
    outline='Failure: %d (longuest=%ds)  Success: %d (longuest=%ds) Skipped: %d' % (failure, longuest_f, success, longuest_s, skipped)
    print(outline)
    if output:
        print(outline, file=output)

last_sigint = None

def sigint_handler(sig, frame):
    global last_sigint
    if last_sigint and time.time()<last_sigint+3:
        last_sigint = time.time() 
        raise KeyboardInterrupt('raised from insight the sigint_handler')
    print(" showing status, press Ctrl+C again to abort test")
    show_resume()
    lbl=""
    if label:
        if len(prevails)>1:
            lbl="%d/%d %s" % ( ilabel, len(prevails), label)
        else:
            lbl=label
        
    print('loop %d | %s | test %d/%d %s | failed %d' % (iloop, lbl, itest, len(alltests), test, failed))
    print()
    last_sigint = time.time()

def bacula_backtrace(output=None):
    print('BACKTRACE BACKTRACE BACKTRACE BACKTRACE BACKTRACE BACKTRACE BACKTRACE')
    proc=subprocess.Popen([ './bta', ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
    out, err=proc.communicate()
    for line in out.splitlines():
        print(line)
        if output:
            print(line, file=output)

parser = argparse.ArgumentParser(description='Bacula regress script front-end')
parser.add_argument('tests', metavar='TESTS', type=str, nargs='+', help='list of script to run')
parser.add_argument('--dedup', dest='dedup', action='store', help='choose dedup mode', choices='bothsides storage none disable all'.split())
parser.add_argument('--dedup-cache', dest='dedup_cache', action='store', help='enable or disable client cache', choices='yes no all'.split())
parser.add_argument('-i', '--ini', dest='ini', action='store', help='ini file', default='arun.ini')
parser.add_argument('-c', '--config', dest='config', action='store', help='configuration file', default='config')
parser.add_argument('-O', '--reset-output', dest='reset_output', action='store_true', help='reset output file')
parser.add_argument('-o', '--output', dest='output', action='store', help='output file', default='test.out')
parser.add_argument('-v', '--verbose', dest='verbose', action='count', default=0, help='make run more verbose')
parser.add_argument('-d', '--debug', dest='debug', action='count', default=0, help='debug mode')
parser.add_argument('-q', '--quick', dest='quick', action='store_true', help='quick commandline in section [QUICK]')
parser.add_argument('-l', '--loop', dest='loop', action='store_true', help='loop forever')
parser.add_argument('-e', '--error-stop', dest='error_stop', action='store_true', help='stop on first error')
parser.add_argument('-b', '--build', dest='build', action='store_true', help='build the source if required')
parser.add_argument('-B', '--re-build', dest='re_build', action='store_true', help='do a make setup')
parser.add_argument('-w', '--build-win32', dest='build_win32', action='store_true', help='build win32 64bits client')
parser.add_argument('-x', '--xparam', dest='xparam', action='store_true', help='run the script for every possible xparam values')
parser.add_argument('--xparam-level', dest='xparam_level', type=int, help='ignore XPARAM values above the given level', default='9')
parser.add_argument('--warn-time', dest='warn_time', type=int, action='store', help='timeout in sec before to send a SIGTERM', default=600)
parser.add_argument('--kill-time', dest='kill_time', type=int, action='store', help='timeout in sec after the warn-time before to send a SIGKILL', default=50)

args = parser.parse_args()

ini = configparser.ConfigParser()
ini.optionxform = str # don't convert option to lowercase
try:
    ini.read(args.ini)
except ConfigParser.ParsingError:
    print("\nDON'T FORGET TO ADD A ':' AT THE END OF YOUR REGRESS SCRIPT !!!\n")
    raise
if args.quick:
    try:
        name=args.tests[0]
        cmdline=ini.get('QUICK', name)
    except ConfigParser.NoSectionError:
        parser.error('section [QUICK] not found in "%s"' % (args.ini, ))
    except ConfigParser.NoOptionError:
        parser.error('option "%s" not found in section [QUICK]' % (name, ))
    else:
        print('using:', cmdline)
        cmd=cmdline.split()
        if args.debug and not '-d' in cmd:
            cmd.insert(0, '-d')

        args = parser.parse_args(cmd)

# get the content of 'config'
config_content = open(args.config).read()
config_content+='\nset\n'
proc=subprocess.Popen([ '/bin/sh', ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err=proc.communicate(codecs.encode(config_content))
config=parse_shell_string(codecs.decode(out))

prevails=[ ]

verbose=args.verbose

base_env=os.environ.copy()

if config.get('FORCE_DEDUP', 'no')=='no':
    if args.dedup=='disable' or args.dedup=='all' or args.dedup==None:
        prevail=dict()
        prevail['FORCE_DEDUP']='no'
        prevail['DEDUP_FS_OPTION']=None # delete the environment
        prevail['DEDUP_FD_CACHE']=None
        prevails.append(('disable', prevail))
    else:
        print("FORCE_DEDUP is 'no', '--dedup disable' is the only valid option", file=sys.stderr)
        sys.exit(1)
else: 
    if args.dedup=='disable':
        print("FORCE_DEDUP is 'yes', '--dedup disable' is an invalid option", file=sys.stderr)
        sys.exit(1)
    
    if args.dedup==None:
        args.dedup=config.get('DEDUP_FS_OPTION', 'bothsides')

    if args.dedup_cache==None:
        args.dedup_cache=config.get('DEDUP_FD_CACHE', 'no')


for mode in [ 'bothsides', 'storage', 'none']:
    if config.get('FORCE_DEDUP', 'no')=='yes' and (args.dedup==mode or args.dedup=='all'):
        prevail=dict()
        prevail['FORCE_DEDUP']='yes'
        prevail['DEDUP_FS_OPTION']=mode
        if args.dedup_cache!=None:
            prevail['DEDUP_FD_CACHE']=args.dedup_cache
        prevails.append((mode, prevail))

if args.dedup_cache=='all':
    lst=[]
    for dedup_cache in [ 'yes', 'no']:
        for mode, prevail in prevails:
            prevail['DEDUP_FD_CACHE']=dedup_cache
            lst.append((mode+'-'+'fd_cache='+dedup_cache, prevail.copy()))
    prevails=lst

if args.debug:
    base_env['REGRESS_DEBUG']='1'

if verbose:
    print('Prevails', prevails)
    print('tests', args.tests)
 
alltests=[]

for test in args.tests:
    if test.startswith('@'):
        if ini.has_section(test):
            for option in ini.options(test):
                filename=option
                st=ini.get(test, option)
                argst=parse_arg_string(st)
                if argst==None:
                    print("cannot parse argument string for test %s:%s skip (\"%s\")" % (option, test, argst), file=sys.stderr)
                    continue
                if not os.path.dirname(filename):
                    filename=os.path.join('tests', filename)
                if not os.path.isfile(filename):
                    print("%s: test file not found in %s, skip" % (option, test), file=sys.stderr)
                else:
                    alltests.append((filename, argst))
        else:
            print("%s: section not found, skip" % (test, ), file=sys.stderr)
            
    else:
        try:
            filename, st=test.split(':', 1)
            argst=parse_arg_string(st)            
            if argst==None:
                print("cannot parse argument string for test %s:%s skip (\"%s\")" % (option, test, argst), file=sys.stderr)
                continue
        except ValueError:
            filename, argst=test, dict()
            
        if not os.path.dirname(filename):
            filename=os.path.join('tests', filename)
        if not os.path.isfile(filename):
            print("%s: test file not found, skip" % (filename, ), file=sys.stderr)
        elif not os.access(filename, os.X_OK):
            print("%s: test file not executable, skip" % (filename, ), file=sys.stderr)
        else:
            alltests.append((filename, argst))
            
if verbose:
    print('all tests:', alltests)

if args.build:
    proc=subprocess.Popen([ 'scripts/async.sh', ], stdin=subprocess.PIPE)
    out, err=proc.communicate() # out, & err are binary not string
    if proc.returncode!=0:
        sys.exit(proc.returncode)
elif args.re_build:
    proc=subprocess.Popen([ 'make', 'setup', ], stdin=subprocess.PIPE)
    out, err=proc.communicate() # out, & err are binary not string
    if proc.returncode!=0:
        sys.exit(proc.returncode)

if args.build_win32:
    BACULA_SOURCE=config.get('BACULA_SOURCE')
    win32_source=os.path.join(BACULA_SOURCE, 'src', 'win32')
    proc=subprocess.Popen([ 'make', '-C', win32_source, 'win64=yes', 'bat=no', 'winfiled' ], stdin=subprocess.PIPE)
    out, err=proc.communicate() # out, & err are binary not string
    if proc.returncode!=0:
        sys.exit(proc.returncode)
    # upload the binary
    url='http://{}:8091/install?truncate_traces={}'.format(config.get('WIN32_ADDR'), 'yes' if args.reset_output else 'no')
    proc=subprocess.Popen([ 'wget', '-qO', 'tmp/win32_install.log', url ], stdin=subprocess.PIPE)
    # NOTICE the tmp/win32_install.log is deleted by the "cleanup" of each test
    out, err=proc.communicate() # out, & err are binary not string
    if proc.returncode!=0:
        sys.exit(proc.returncode)

output=open(args.output, 'w' if args.reset_output else 'a' , 1)
reports=[]
skipped=0
failed=0
iloop=0
try:
    signal.signal(signal.SIGINT, sigint_handler)
    while alltests:
        iloop+=1
        for ilabel, (label, prevail) in enumerate(prevails):
            if label and (len(prevails)>1 or args.dedup=='all' or args.dedup_cache=='all'):
                print()
                print("=== LABEL %s ===" % (label ))
                print()
                print(file=output)
                print("=== LABEL %s ===" % (label ), file=output)
                print(file=output)
        
            env=base_env.copy()
            
            for k, v in iter(prevail.items()):
                if v==None:
                    if k in env:
                        del env[k]
                else:
                    env['PREVAIL_'+k]=v
            
            if verbose:
                print('env:', env)
            
            for itest, (test, option) in enumerate(alltests):
                warn_time=args.warn_time
                kill_time=args.kill_time
                test_env=env.copy()
                test_env['WARN_TIME']=str(warn_time)
                test_env['KILL_TIME']=str(kill_time)
                xparams=dict()
                if args.xparam:
                    for line in open(test, 'r'):
                        match=re_test_param.match(line)
                        if match:
                            param, value=match.group('param', 'value')
                            if param=='WARN_TIME':
                                warn_time=int(value)
                            elif param=='KILL_TIME':
                                kill_time=int(value)
                            test_env[param]=value
                            if verbose:
                                print("PARAM %s=%s" %(param, value))
                            continue
                        match=re_test_xparam.match(line)
                        if match:
                            xparam, values=match.group('xparam', 'values')
                            if values.startswith('<') and values.endswith('>'):
                                continue
                            if values.startswith('(') and values.endswith(')'):
                                values=values[1:-1].split('|')
                            else:
                                values=[values, ]
                            xparams[xparam]=[]
                            for value in values:
                                if value.startswith('<') and value.endswith('>'):
                                    continue
                                match=re_test_xparam_value.match(value)
                                level, value=match.group('level', 'value')
                                try:
                                    level=int(level)
                                except (TypeError, ValueError):
                                    level=0
                                if  level<=args.xparam_level:
                                    xparams[xparam].append(value)

                if not xparams or not args.xparam:
                    xparams[None]=[None]

                if 'warning' in option:
                    warning=option['warning']
                    print("WARNING:", warning, file=sys.stderr)
                    print("WARNING:", warning, file=output)
                
                if 'skip' in option:
                    skipped+=1
                    if option.get('skip')!=None:
                        print('skip: %s%s' % (test, "" if option.get('skip')==None else ' "'+option.get('skip')+'"'), file=sys.stderr)
                        print('skip: %s%s' % (test, "" if option.get('skip')==None else ' "'+option.get('skip')+'"'), file=output)
                    continue

                for xparam in mixer(xparams):
                    # print(xparam)
                    xparam_label=[]
                    for k in xparam:
                        if k!=None:
                            test_env[k]=xparam[k]
                            xparam_label.append('%s=%s' % (k, xparam[k]))

                    test_env['XPARAM_LABEL']=' '.join(xparam_label)

                    try:
                        # universal_newlines=True force sdt(in|out|err) to be opened as text
                        process=subprocess.Popen([ test, ], env=test_env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setpgrp, universal_newlines=True)
                    except Exception as e:
                        print("cmdline=%r" % ([ test, ], ), file=sys.stderr)
                        print("env=%r" % (test_env, ), file=sys.stderr)
                        raise
                    
                    try:
                        reader=NonBlockingReader(process.stdout)
                        start=time.time()
                        while process.poll() is None:
                            try:
                                line=reader.readline(1.0)
                            except NonBlockingReader.Timeout:
                                # timeout
                                if warn_time>0 and time.time()>start+warn_time:
                                    try:
                                        timeout=True;
                                        process.terminate()
                                    except OSError:
                                        pass                   
                                    while kill_time>0 and process.poll() is None and time.time()<start+warn_time+kill_time:
                                        try:
                                            line=reader.readline(1.0)
                                        except NonBlockingReader.Timeout:
                                            continue
                                        else:
                                            if line:
                                               print(line)
                                               print(line, file=output)
                                    if process.poll() is None:
                                        bacula_backtrace(output)
                                        if not args.error_stop:
                                            try:
                                                process.kill()
                                            except OSError:
                                                pass
                                        else:
                                            """dont kill, let the process survive and the user inspect it"""
                                    break
                            else:
                                if line:
                                    print(line, end='')
                                    print(line, file=output, end='')
    
                        # Display lines remaining in the buffer of the reader           
                        while True:
                            try:
                                line=reader.readline()
                            except queue.Empty:
                                break
                            else:
                                if line:
                                    print(line, end='')
                                    print(line, file=output, end='')
                                    
                    except KeyboardInterrupt:
                        print(" test \"%s\" aborted" % (test, ))
                        print("Press Ctrl+C one more time to stop")
                        process.kill()
                        process.wait()
                        process.returncode=Status['abort']
                        time.sleep(3)
                        
                    reports.append((test, label, prevail, option, process.returncode, time.time()-start))
                    if process.returncode!=0:
                        failed+=1
                        print()
                        print("=== Test %s failed ===" % (test ))
                        print()
                        print(file=output)
                        print("=== Test %s failed ===" % (test ), file=output)
                        print(file=output)
                        if args.error_stop:
                            raise StopOnFirstError('first error')

        if not args.loop:
            break
except KeyboardInterrupt:
    print(" Stopping...")
except StopOnFirstError:
    print("Stop on first error enable")

show_resume(output)

output.close()

if args.build_win32:
    # download the win32 trace file
    url='http://{}:8091/get_traces'.format(config.get('WIN32_ADDR'))
    proc=subprocess.Popen([ 'wget', '-qO', 'tmp/windows-fd.trace', url ], stdin=subprocess.PIPE)
    out, err=proc.communicate() # out, & err are binary not string
