From fee4186f4ad11ec78f745d13665e45fd84625420 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 9 Mar 2021 00:55:04 -0500 Subject: Convert to package --- .gitignore | 10 +++ LICENSE.txt | 29 ++++++++ README.md | 1 + jenkins_log_raider.py | 185 ---------------------------------------------- log_raider/__init__.py | 1 + log_raider/jenkins.py | 197 +++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 6 ++ setup.py | 21 ++++++ 8 files changed, 265 insertions(+), 185 deletions(-) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 README.md delete mode 100644 jenkins_log_raider.py create mode 100644 log_raider/__init__.py create mode 100644 log_raider/jenkins.py create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..29cc18b --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.pyc +__pycache__ +.idea +build +dist +*.egg-info +pip-* +venv +version.py +consoleText* diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..63c8315 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, Association of Universities for Research in Astronomy +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1ab17c1 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# Log Raider diff --git a/jenkins_log_raider.py b/jenkins_log_raider.py deleted file mode 100644 index 6fd9126..0000000 --- a/jenkins_log_raider.py +++ /dev/null @@ -1,185 +0,0 @@ -import argparse -import os -import sys - - -PIPELINE_VERBOSE = False -PIPELINE_MARKER = "[Pipeline]" -PIPELINE_BLOCK_START = '{' -PIPELINE_BLOCK_END = '}' -PIPELINE_BLOCK_START_NOTE = '(' -PIPELINE_BLOCK_START_NOTE_END = ')' -PIPELINE_BLOCK_END_NOTE = "//" - - -def consume_token(s, token, l_trunc=0, r_trunc=0, strip_leading=True): - """ - :param s: String to scan - :param token: Substring to target - :param l_trunc: Truncate chars from left; Positive integer - :param r_trunc: Truncate chars from right; Positive integer - :param strip_leading: Remove leading whitespace - :return: Modified input string - """ - s_len = len(s) - token_len = len(token) - result = s[l_trunc + s.find(token) + token_len:s_len - r_trunc] - - if strip_leading: - result = result.strip() - - return result - - -def printv(*args, **kwargs): - if PIPELINE_VERBOSE: - print(*args, *kwargs) - - -def parse_log(filename): - """ - Parse a Jenkins pipeline log and return information about each logical section - :param filename: path to log file - :return: list of section dictionaries - """ - section_defaults = { - "type": "", - "name": "", - "data": "", - "depth": 0, - } - depth = 0 - master_log = False - last_name = "" - last_type = "" - section = section_defaults.copy() - result = list() - - # Reducing code duplication and trying to increase readability with "macro" like functions here. - # Don't move these. - def is_pipeline(lobj): - return lobj[0].startswith(PIPELINE_MARKER) - - def is_pipeline_block_start(lobj): - return lobj[0] == PIPELINE_BLOCK_START - - def is_pipeline_block_start_note(lobj): - return lobj[0].startswith(PIPELINE_BLOCK_START_NOTE) - - def is_pipeline_block_end(lobj): - return lobj[0] == PIPELINE_BLOCK_END - - def is_pipeline_block_end_note(lobj): - return lobj[0] == PIPELINE_BLOCK_END_NOTE - - def commit(block): - if block != section_defaults: - if not block["name"]: - block["name"] = last_name - if not block["type"]: - block["type"] = last_type - block["depth"] = depth - block["line"] = lineno - result.append(block) - block = section_defaults.copy() - return block - - # Parsing begins - for lineno, line in enumerate(open(filename, "r").readlines()): - line = line.strip() - if not line: - continue - - # Format: - # [Pipeline]\ {?}?\ (?STRING?|\/\/\ COMMENT?)? - # - # The first two arguments are important. - # 1) Is it a pipeline log record? - # 2) Is it a block start/end/comment? - # All string data beyond this point is considered a section "name" or "type" - data = line.split(" ", 2) - - if is_pipeline(data): - # Parsing: [Pipeline] - # Information related to the groovy pipeline is always preceded by: [Pipeline] - if master_log: - section = commit(section) - - master_log = False - section = commit(section) - data = data[1:] - - if is_pipeline_block_end(data): - # Parsing: [Pipeline] } - depth -= 1 - printv("DEPTH ::", depth) - if section != section_defaults: - section = commit(section) - elif is_pipeline_block_end_note(data): - # Ignoring: [Pipeline] // NAME HERE - pass - elif is_pipeline_block_start(data): - # Parsing: [Pipeline] { - depth += 1 - printv("DEPTH ::", depth) - - if len(data) == 2: - # Parsing: [Pipeline] { (NAME HERE) - # Read the stage name. - # Stage names are preceded by a "{" and the literal name is encapsulated by parenthesis - x = data[1:] - if is_pipeline_block_start_note(x): - section["name"] = consume_token(x[0], PIPELINE_BLOCK_START_NOTE, r_trunc=1) - last_name = section["name"] - printv("NAME ::", section["name"]) - elif len(data) == 1: - # Parsing: [Pipeline] NAME HERE - # A standalone string without a preceding "{" denotes the type of step being executed - section["type"] = data[0] - last_type = section["type"] - printv("TYPE ::", section["type"]) - - # Finished with [Pipeline] data for now... see if there's more - continue - - if not depth and not is_pipeline(line): - # Parser depth begins at zero. Trigger only when a line doesn't begin with: [Pipeline] - master_log = True - section["name"] = "master" - section["type"] = "masterLog" - section["data"] += line + "\n" - section["depth"] = depth - else: - # Consume raw log data at current depth - section["type"] = last_type - section["data"] += line + "\n" - printv("RAW ::", line) - - # Save any data appearing after: [Pipeline] End of Pipeline - # This data belongs to the master node - commit(section) - - return result - - -def main(): - global PIPELINE_VERBOSE - parser = argparse.ArgumentParser() - parser.add_argument("logfile") - parser.add_argument("-v", "--verbose", help="Increase verbosity") - args = parser.parse_args() - - if not os.path.exists(args.logfile): - print(f"{args.logfile}: does not exist") - - PIPELINE_VERBOSE = True - data = parse_log(args.logfile) - for x in data: - print("#" * 79) - print(f"{x['type']} - {x['name']} (line @ {x['line']})") - print("#" * 79) - print(f"{x['data']}") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/log_raider/__init__.py b/log_raider/__init__.py new file mode 100644 index 0000000..87fbdb0 --- /dev/null +++ b/log_raider/__init__.py @@ -0,0 +1 @@ +from .version import version as __version__ diff --git a/log_raider/jenkins.py b/log_raider/jenkins.py new file mode 100644 index 0000000..16096a5 --- /dev/null +++ b/log_raider/jenkins.py @@ -0,0 +1,197 @@ +import argparse +import json +import os +import sys +from . import __version__ + +PIPELINE_VERBOSE = False +PIPELINE_MARKER = "[Pipeline]" +PIPELINE_BLOCK_START = '{' +PIPELINE_BLOCK_END = '}' +PIPELINE_BLOCK_START_NOTE = '(' +PIPELINE_BLOCK_START_NOTE_END = ')' +PIPELINE_BLOCK_END_NOTE = "//" + + +def consume_token(s, token, l_trunc=0, r_trunc=0, strip_leading=True): + """ + :param s: String to scan + :param token: Substring to target + :param l_trunc: Truncate chars from left; Positive integer + :param r_trunc: Truncate chars from right; Positive integer + :param strip_leading: Remove leading whitespace + :return: Modified input string + """ + s_len = len(s) + token_len = len(token) + result = s[l_trunc + s.find(token) + token_len:s_len - r_trunc] + + if strip_leading: + result = result.strip() + + return result + + +def printv(*args, **kwargs): + if PIPELINE_VERBOSE: + print(*args, *kwargs) + + +def parse_log(filename): + """ + Parse a Jenkins pipeline log and return information about each logical section + :param filename: path to log file + :return: list of section dictionaries + """ + section_defaults = { + "type": "", + "name": "", + "data": "", + "depth": 0, + } + depth = 0 + master_log = False + last_name = "" + last_type = "" + section = section_defaults.copy() + result = list() + + # Reducing code duplication and trying to increase readability with "macro" like functions here. + # Don't move these. + def is_pipeline(lobj): + return lobj[0].startswith(PIPELINE_MARKER) + + def is_pipeline_block_start(lobj): + return lobj[0] == PIPELINE_BLOCK_START + + def is_pipeline_block_start_note(lobj): + return lobj[0].startswith(PIPELINE_BLOCK_START_NOTE) + + def is_pipeline_block_end(lobj): + return lobj[0] == PIPELINE_BLOCK_END + + def is_pipeline_block_end_note(lobj): + return lobj[0] == PIPELINE_BLOCK_END_NOTE + + def commit(block): + if block != section_defaults: + if not block["name"]: + block["name"] = last_name + if not block["type"]: + block["type"] = last_type + block["depth"] = depth + block["line"] = lineno + result.append(block) + block = section_defaults.copy() + return block + + # Parsing begins + for lineno, line in enumerate(open(filename, "r").readlines()): + line = line.strip() + if not line: + continue + + # Format: + # [Pipeline]\ {?}?\ (?STRING?|\/\/\ COMMENT?)? + # + # The first two arguments are important. + # 1) Is it a pipeline log record? + # 2) Is it a block start/end/comment? + # All string data beyond this point is considered a section "name" or "type" + data = line.split(" ", 2) + + if is_pipeline(data): + # Parsing: [Pipeline] + # Information related to the groovy pipeline is always preceded by: [Pipeline] + if master_log: + section = commit(section) + + master_log = False + section = commit(section) + data = data[1:] + + if is_pipeline_block_end(data): + # Parsing: [Pipeline] } + depth -= 1 + printv("DEPTH ::", depth) + if section != section_defaults: + section = commit(section) + elif is_pipeline_block_end_note(data): + # Ignoring: [Pipeline] // NAME HERE + pass + elif is_pipeline_block_start(data): + # Parsing: [Pipeline] { + depth += 1 + printv("DEPTH ::", depth) + + if len(data) == 2: + # Parsing: [Pipeline] { (NAME HERE) + # Read the stage name. + # Stage names are preceded by a "{" and the literal name is encapsulated by parenthesis + x = data[1:] + if is_pipeline_block_start_note(x): + section["name"] = consume_token(x[0], PIPELINE_BLOCK_START_NOTE, r_trunc=1) + last_name = section["name"] + printv("NAME ::", section["name"]) + elif len(data) == 1: + # Parsing: [Pipeline] NAME HERE + # A standalone string without a preceding "{" denotes the type of step being executed + section["type"] = data[0] + last_type = section["type"] + printv("TYPE ::", section["type"]) + + # Finished with [Pipeline] data for now... see if there's more + continue + + if not depth and not is_pipeline(line): + # Parser depth begins at zero. Trigger only when a line doesn't begin with: [Pipeline] + master_log = True + section["name"] = "master" + section["type"] = "masterLog" + section["data"] += line + "\n" + section["depth"] = depth + else: + # Consume raw log data at current depth + section["type"] = last_type + section["data"] += line + "\n" + printv("RAW ::", line) + + # Save any data appearing after: [Pipeline] End of Pipeline + # This data belongs to the master node + commit(section) + + return result + + +def main(): + global PIPELINE_VERBOSE + parser = argparse.ArgumentParser() + parser.add_argument("-j", "--json", action="store_true", help="Emit JSON") + parser.add_argument("-v", "--verbose", action="store_true", help="Increase verbosity") + parser.add_argument("-V", "--version", action="store_true", help="Show version") + parser.add_argument("logfile", nargs='?') + + args = parser.parse_args() + + if args.version: + print(__version__) + exit(0) + + if not os.path.exists(args.logfile): + print(f"{args.logfile}: does not exist") + + PIPELINE_VERBOSE = args.verbose + data = parse_log(args.logfile) + + if args.json: + print(json.dumps(data, indent=2)) + else: + for x in data: + print("#" * 79) + print(f"{x['type']} - {x['name']} (line @ {x['line']})") + print("#" * 79) + print(f"{x['data']}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4a3eab1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=30.3.0", + "wheel", + "setuptools_scm" +] diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1f72777 --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +from setuptools import setup +from setuptools import find_packages + +setup( + name="log_raider", + use_scm_version={ + "write_to": "log_raider/version.py", + }, + packages=find_packages(), + setup_requires=[ + "setuptools_scm", + ], + entry_points={ + "console_scripts": [ + "log_raider_jenkins = log_raider.jenkins:main", + ], + }, + author="Joseph Hunkeler", + author_email="jhunk@stsci.edu", + url="https://github.com/spactelescope/log_raider", +) -- cgit