diff --git a/pyproject.toml b/pyproject.toml index 092e072..79c44e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ dependencies = [ "numpy", "pandas", "parmap", - "psycopg2-binary", "python-dateutil", "py-tlsh", "pytz", diff --git a/src/fosslight_binary/_binary_dao.py b/src/fosslight_binary/_binary_dao.py index 17932e1..4b34a4c 100755 --- a/src/fosslight_binary/_binary_dao.py +++ b/src/fosslight_binary/_binary_dao.py @@ -2,185 +2,278 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020 LG Electronics Inc. # SPDX-License-Identifier: Apache-2.0 +"""Binary DB lookup via ldb_service POST /binary/match.""" -import tlsh +import json import logging -import psycopg2 -import pandas as pd -import socket +import os +import urllib.error +import urllib.request +from typing import Dict, List, Optional, Tuple from urllib.parse import urlparse + from fosslight_binary._binary import TLSH_CHECKSUM_NULL from fosslight_util.oss_item import OssItem import fosslight_util.constant as constant -columns = ['filename', 'pathname', 'checksum', 'tlshchecksum', 'ossname', - 'ossversion', 'license', 'platformname', - 'platformversion'] -conn = "" -cur = "" logger = logging.getLogger(constant.LOGGER_NAME) -DB_URL_DEFAULT = "postgresql://bin_analysis_script_user:script_123@bat.lge.com:5432/bat" +DEFAULT_KB_URL = "http://fosslight-kb.lge.com/" +_BINARY_MATCH_PATH = "/binary/match" +_HTTP_TIMEOUT_SEC = 120 +_PROBE_TIMEOUT_SEC = 10 +_DEFAULT_CHUNK_SIZE = 3000 -def get_oss_info_from_db(bin_info_list, dburl=""): - _cnt_auto_identified = 0 - conn_str, dbc = get_connection_string(dburl) - # DB URL에서 host 추출 + +def _get_chunk_size() -> int: + raw = os.environ.get("BINARY_MATCH_CHUNK_SIZE", str(_DEFAULT_CHUNK_SIZE)) try: - db_host = dbc.hostname - except Exception as ex: - logger.warning(f"Failed to parse DB URL for host: {ex}") - db_host = None + chunk_size = int(raw) + except ValueError: + logger.warning(f"Invalid BINARY_MATCH_CHUNK_SIZE={raw!r}; using {_DEFAULT_CHUNK_SIZE}") + return _DEFAULT_CHUNK_SIZE + if chunk_size <= 0: + logger.warning( + f"BINARY_MATCH_CHUNK_SIZE must be > 0 (got {chunk_size}); using {_DEFAULT_CHUNK_SIZE}" + ) + return _DEFAULT_CHUNK_SIZE + return chunk_size + + +_CHUNK_SIZE = _get_chunk_size() + +MatchKey = Tuple[str, str] + + +def resolve_kb_config(kb_url: str = "", kb_token: str = "") -> Tuple[str, str]: + url = (kb_url or os.environ.get("KB_URL", DEFAULT_KB_URL)).strip() or DEFAULT_KB_URL + token = (kb_token or "").strip() or (os.environ.get("KB_TOKEN") or "").strip() + return f"{url.rstrip('/')}/", token + + +def _is_valid_kb_url_format(kb_url: str) -> bool: + parsed = urlparse(kb_url) + return parsed.scheme in ("http", "https") and bool(parsed.netloc) + + +def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> Tuple[bool, str]: + """ + Return (available, cover_comment). + + cover_comment: + - ``KB({kb_url}) Unreachable`` when the host cannot be reached + - ``KB({kb_url}) Skipped`` when /binary/match is missing (HTTP 404) + - empty for other skip cases (invalid URL) + """ + if not _is_valid_kb_url_format(kb_url): + logger.warning(f"Invalid KB URL format: {kb_url}") + return False, "" + + endpoint = f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}" + data = json.dumps({"items": []}).encode("utf-8") + request = urllib.request.Request(endpoint, data=data, method="POST") + request.add_header("Accept", "application/json") + request.add_header("Content-Type", "application/json") + if kb_token: + request.add_header("Authorization", f"Bearer {kb_token}") - is_internal = False - if db_host: - try: - # DNS lookup 시도 - socket.gethostbyname(db_host) - is_internal = True - except Exception: - is_internal = False - - if is_internal: - connect_to_lge_bin_db(conn_str) - if conn != "" and cur != "": - for item in bin_info_list: - bin_oss_items = [] - tlsh_value = item.tlsh - checksum_value = item.checksum - bin_file_name = item.binary_name_without_path - - df_result = get_oss_info_by_tlsh_and_filename( - bin_file_name, checksum_value, tlsh_value) - if df_result is not None and len(df_result) > 0: - _cnt_auto_identified += 1 - # Initialize the saved contents at .jar analyzing only once - if not item.found_in_jar_analysis and item.oss_items: - item.oss_items = [] - - for idx, row in df_result.iterrows(): - if not item.found_in_jar_analysis: - oss_from_db = OssItem(row['ossname'], row['ossversion'], row['license']) - - if bin_oss_items: - if not any(oss_item.name == oss_from_db.name - and oss_item.version == oss_from_db.version - and oss_item.license == oss_from_db.license - for oss_item in bin_oss_items): - bin_oss_items.append(oss_from_db) - else: - bin_oss_items.append(oss_from_db) - - if bin_oss_items: - item.set_oss_items(bin_oss_items) - item.comment = "Binary DB result" - item.found_in_binary = True - else: - logger.warning(f"Internal network detected, but DB connection to '{db_host}' failed. Skipping DB query.") - disconnect_lge_bin_db() - else: - logger.debug(f"Binary DB host '{db_host}' is not reachable. Skipping DB query.") - return bin_info_list, _cnt_auto_identified - - -def get_connection_string(dburl): - # dburl format : 'postgresql://username:password@host:port/database_name' - connection_string = "" - user_dburl = True - dbc = "" - if dburl == "" or dburl is None: - user_dburl = False - dburl = DB_URL_DEFAULT try: - if user_dburl: - logger.debug("DB URL:" + dburl) - dbc = urlparse(dburl) - connection_string = "dbname={dbname} user={user} host={host} password={password} port={port}" \ - .format(dbname=dbc.path.lstrip('/'), - user=dbc.username, - host=dbc.hostname, - password=dbc.password, - port=dbc.port) + with urllib.request.urlopen(request, timeout=_PROBE_TIMEOUT_SEC) as response: + response.read() + return True, "" + except urllib.error.HTTPError as ex: + if ex.code == 404: + logger.warning( + f"KB binary match endpoint not found (HTTP 404): {endpoint}. " + "Skipping binary match API." + ) + return False, f"KB({kb_url}) Skipped" + logger.debug( + f"KB binary match endpoint responded with HTTP {ex.code}; treated as available" + ) + return True, "" + except urllib.error.URLError as ex: + logger.warning(f"KB binary match endpoint unreachable: {ex}. Skipping binary match API.") + return False, f"KB({kb_url}) Unreachable" except Exception as ex: - if user_dburl: - logger.warning(f"(Minor) Failed to parsing db url : {ex}") - - return connection_string, dbc - - -def get_oss_info_by_tlsh_and_filename(file_name, checksum_value, tlsh_value): - sql_statement = "SELECT filename,pathname,checksum,tlshchecksum,ossname,ossversion,\ - license,platformname,platformversion FROM lgematching " - sql_statement_checksum = " WHERE filename=%s AND checksum=%s;" # Using parameterized query - sql_statement_filename = "SELECT DISTINCT ON (tlshchecksum) tlshchecksum FROM lgematching WHERE filename=%s;" # Using parameterized query - - final_result_item = "" - - df_result = get_list_by_using_query( - sql_statement + sql_statement_checksum, columns, (file_name, checksum_value)) - # Found a file with the same checksum. - if df_result is not None and len(df_result) > 0: - final_result_item = df_result - else: - # Match tlsh and fileName - df_result = get_list_by_using_query( - sql_statement_filename, ['tlshchecksum'], (file_name,)) - if df_result is None or len(df_result) <= 0: - final_result_item = "" - elif tlsh_value == TLSH_CHECKSUM_NULL: # Couldn't get the tlsh of a file. - final_result_item = "" - else: - matched_tlsh = "" - matched_tlsh_diff = -1 - for row in df_result.tlshchecksum: - try: - if row != TLSH_CHECKSUM_NULL: - tlsh_diff = tlsh.diff(row, tlsh_value) - if tlsh_diff <= 120: # MATCHED - if (matched_tlsh_diff < 0) or (tlsh_diff < matched_tlsh_diff): - matched_tlsh_diff = tlsh_diff - matched_tlsh = row - except Exception as ex: - logger.warning(f"* (Minor) Error_tlsh_comparison: {ex}") - if matched_tlsh != "": - final_result_item = get_list_by_using_query( - sql_statement + " WHERE filename=%s AND tlshchecksum=%s;", columns, (file_name, matched_tlsh)) - - return final_result_item - - -def get_list_by_using_query(sql_query, columns, params=None): - result_rows = "" # DataFrame - try: - if params: - cur.execute(sql_query, params) + logger.warning( + f"Failed to check KB binary match endpoint: {ex}. Skipping binary match API." + ) + return False, f"KB({kb_url}) Unreachable" + + +def _is_unknown_checksum(checksum: str) -> bool: + """True when checksum was not computed (empty or TLSH_CHECKSUM_NULL).""" + return (not checksum) or checksum == TLSH_CHECKSUM_NULL + + +def _is_unknown_tlsh(tlsh: str) -> bool: + """True when tlsh was not computed (empty or TLSH_CHECKSUM_NULL).""" + return (not tlsh) or tlsh == TLSH_CHECKSUM_NULL + + +def _match_key(filename: str, checksum: str, index: int) -> MatchKey: + """Dedupe key. Unknown checksums stay unique per list index (no filename-only merge).""" + if _is_unknown_checksum(checksum): + return filename, f"__unknown_{index}" + return filename, checksum + + +def _build_deduped_payload( + bin_info_list, + tlsh_null: str, +) -> Tuple[List[dict], Dict[MatchKey, str]]: + """Deduplicate by filename+checksum; return API payload and key→api_id map. + + Items with empty/\"0\" checksum are not deduped — each keeps its own API entry + (and its own tlsh) so distinct files that share a basename are not conflated. + Items with both checksum and tlsh unknown are omitted (nothing to match). + """ + key_to_id: Dict[MatchKey, str] = {} + items_payload: List[dict] = [] + + for index, item in enumerate(bin_info_list): + if item.exclude: + continue + filename = item.binary_name_without_path + checksum = item.checksum or "" + tlsh = item.tlsh or tlsh_null + if _is_unknown_checksum(checksum) and _is_unknown_tlsh(tlsh): + continue + key = _match_key(filename, checksum, index) + if not _is_unknown_checksum(checksum) and key in key_to_id: + continue + api_id = str(len(items_payload)) + key_to_id[key] = api_id + items_payload.append({ + "id": api_id, + "filename": filename, + "checksum": checksum, + "tlsh": tlsh, + }) + + return items_payload, key_to_id + + +def _apply_match_result_to_item(item, result: dict) -> bool: + """Apply a /binary/match result to one binary item. Returns True if matched.""" + if not result or not result.get("matched"): + return False + oss_rows = result.get("oss_items") or [] + if not oss_rows: + return False + + if not item.found_in_jar_analysis and item.oss_items: + item.oss_items = [] + + bin_oss_items = [] + for row in oss_rows: + if item.found_in_jar_analysis: + break + oss_from_db = OssItem( + row.get("oss_name") or "", + row.get("oss_version") or "", + row.get("license") or "", + ) + if bin_oss_items: + if not any( + oss_item.name == oss_from_db.name + and oss_item.version == oss_from_db.version + and oss_item.license == oss_from_db.license + for oss_item in bin_oss_items + ): + bin_oss_items.append(oss_from_db) else: - cur.execute(sql_query) - rows = cur.fetchall() + bin_oss_items.append(oss_from_db) - if rows is not None and len(rows) > 0: - result_rows = pd.DataFrame(data=rows, columns=columns) - except Exception as ex: - logger.error(f"Database query error: {ex}") - result_rows = "" - return result_rows + if bin_oss_items: + item.set_oss_items(bin_oss_items) + item.comment = "Binary DB result" + item.found_in_bin_db = True + return True + return False -def disconnect_lge_bin_db(): - try: - cur.close() - conn.close() - except: - pass +def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): + """ + Call ldb_service /binary/match and attach OSS info to binary items. + Deduplicates by filename+checksum before the API call and maps results back. + Returns (bin_info_list, matched_count, kb_cover_comment). + """ + _cnt_auto_identified = 0 + if not bin_info_list: + return bin_info_list, _cnt_auto_identified, "" + base_url, token = resolve_kb_config(kb_url, kb_token) + available, kb_cover_comment = check_binary_match_endpoint(base_url, token) + if not available: + return bin_info_list, _cnt_auto_identified, kb_cover_comment -def connect_to_lge_bin_db(connection_string): - global conn, cur + items_payload, key_to_id = _build_deduped_payload(bin_info_list, TLSH_CHECKSUM_NULL) + if not items_payload: + return bin_info_list, _cnt_auto_identified, "" + results_by_id = {} try: - conn = psycopg2.connect(connection_string) - cur = conn.cursor() + for chunk_start in range(0, len(items_payload), _CHUNK_SIZE): + chunk = items_payload[chunk_start: chunk_start + _CHUNK_SIZE] + response = _post_binary_match(base_url, token, chunk) + if response is None: + logger.warning( + f"Binary match chunk failed " + f"({chunk_start}:{chunk_start + len(chunk)}); " + "keeping results so far and continuing with next chunks." + ) + continue + for result in response.get("results", []): + results_by_id[str(result.get("id"))] = result except Exception as ex: - logger.debug(f"(Minor) Can't connect to Binary DB. : {ex}") - conn = "" - cur = "" + logger.warning(f"Binary match API failed: {ex}") + + for index, item in enumerate(bin_info_list): + if item.exclude: + continue + key = _match_key(item.binary_name_without_path, item.checksum or "", index) + api_id = key_to_id.get(key) + if api_id is None: + continue + result = results_by_id.get(api_id) + if _apply_match_result_to_item(item, result): + _cnt_auto_identified += 1 + + return bin_info_list, _cnt_auto_identified, "" + + +def _post_binary_match(kb_url: str, kb_token: str, items: list) -> Optional[dict]: + data = json.dumps({"items": items}).encode("utf-8") + request = urllib.request.Request( + f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}", + data=data, + method="POST", + ) + request.add_header("Accept", "application/json") + request.add_header("Content-Type", "application/json") + if kb_token: + request.add_header("Authorization", f"Bearer {kb_token}") + + try: + with urllib.request.urlopen(request, timeout=_HTTP_TIMEOUT_SEC) as response: + body = response.read().decode() + return json.loads(body) if body else {} + except urllib.error.HTTPError as ex: + body = "" + try: + body = ex.read().decode() + except Exception: + pass + if ex.code == 404: + logger.warning( + f"Binary match endpoint not found (HTTP 404): " + f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}" + ) + else: + logger.warning(f"Binary match HTTP {ex.code}: {body or ex.reason}") + return None + except urllib.error.URLError as ex: + logger.debug(f"Binary match unreachable: {ex}") + return None diff --git a/src/fosslight_binary/_help.py b/src/fosslight_binary/_help.py index 8a9ef48..7045551 100644 --- a/src/fosslight_binary/_help.py +++ b/src/fosslight_binary/_help.py @@ -32,7 +32,8 @@ 🔍 Scanner-Specific Options ──────────────────────────────────────────────────────────────────── - -d DB Connection (format: 'postgresql://user:pass@host:port/db') + --kb_url KB API URL (priority: parameter > KB_URL env > default) + --kb_token KB bearer token (priority: parameter > KB_TOKEN env) --notice Print the open source license notice text --no_correction Skip OSS information correction with sbom-info.yaml --correct_fpath Path to custom sbom-info.yaml file @@ -48,8 +49,8 @@ # Generate output in specific format fosslight_binary -f excel -o results/ - # Connect to Binary DB for OSS information - fosslight_binary -d "postgresql://user:pass@localhost:5432/exampledb" + # Binary DB lookup via ldb_service + fosslight_binary --kb_url http://fosslight-kb.lge.com/ --kb_token """ diff --git a/src/fosslight_binary/binary_analysis.py b/src/fosslight_binary/binary_analysis.py index 7060e1e..b883ad1 100755 --- a/src/fosslight_binary/binary_analysis.py +++ b/src/fosslight_binary/binary_analysis.py @@ -169,7 +169,7 @@ def get_file_list(path_to_find, excluded_files): return file_cnt, bin_list, found_jar -def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=False, +def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="", simple_mode=False, correct_mode=True, correct_filepath="", path_to_exclude=[], all_exclude_mode=()): global start_time, finish_time, _root_path, _result_log @@ -254,7 +254,7 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F else: logger.warning("Could not find OSS information for some jar files.") - return_list, db_loaded_cnt = get_oss_info_from_db(return_list, dburl) + return_list, db_loaded_cnt, kb_cover_msg = get_oss_info_from_db(return_list, kb_url, kb_token) return_list = sorted(return_list, key=lambda row: (row.bin_name_with_path)) scan_item.append_file_items(return_list, PKG_NAME) if correct_mode: @@ -267,6 +267,8 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F finish_time = current_timestamp_utc() scan_item.set_cover_comment(f"Detected binaries: {len(return_list)} (Scanned Files : {cnt_file_except_skipped})") + if kb_cover_msg: + scan_item.set_cover_comment(kb_cover_msg) scan_item.set_cover_finish_time(finish_time) for combined_path_and_file, output_extension, output_format in zip(result_reports, output_extensions, formats): diff --git a/src/fosslight_binary/cli.py b/src/fosslight_binary/cli.py index 56683d4..0ac2bba 100644 --- a/src/fosslight_binary/cli.py +++ b/src/fosslight_binary/cli.py @@ -36,7 +36,8 @@ def main(): path_to_exclude = [] output_dir = "" format = [] - db_url = "" + kb_url = "" + kb_token = "" simple_mode = False correct_mode = True @@ -46,9 +47,10 @@ def main(): parser.add_argument('-s', '--simple', action='store_true', required=False, help=argparse.SUPPRESS) parser.add_argument('-p', '--path', type=str, required=False) parser.add_argument('-o', '--output', type=str, required=False) - parser.add_argument('-d', '--dburl', type=str, default='', required=False) parser.add_argument('-f', '--formats', type=str, required=False, nargs="*") parser.add_argument('-e', '--exclude', nargs="*", required=False, default=[]) + parser.add_argument('--kb_url', type=str, required=False, default="") + parser.add_argument('--kb_token', type=str, required=False, default="") parser.add_argument('--notice', action='store_true', required=False) parser.add_argument('--no_correction', action='store_true', required=False) parser.add_argument('--correct_fpath', nargs=1, type=str, required=False) @@ -81,8 +83,11 @@ def main(): if args.output: # -o option output_dir = args.output - if args.dburl: # -d option - db_url = args.dburl + if args.kb_url: + kb_url = args.kb_url + + if args.kb_token: + kb_token = args.kb_token if args.formats: # -f option format = list(args.formats) @@ -113,7 +118,10 @@ def main(): timer.setDaemon(True) timer.start() - find_binaries(path_to_find_bin, output_dir, format, db_url, simple_mode, correct_mode, correct_filepath, path_to_exclude) + find_binaries( + path_to_find_bin, output_dir, format, kb_url, kb_token, + simple_mode, correct_mode, correct_filepath, path_to_exclude, + ) if __name__ == '__main__':