From 4c73e173f0e97ae68444df7d01edd6e973050423 Mon Sep 17 00:00:00 2001 From: "hongliang.yuan" Date: Fri, 5 Dec 2025 09:26:11 +0800 Subject: [PATCH] support ci bs param --- .../conformer/igie/ci/prepare.sh | 7 +- .../conformer/ixrt/ci/prepare.sh | 7 +- .../scripts/infer_conformer_fp16_accuracy.sh | 4 +- .../infer_conformer_fp16_performance.sh | 4 +- tests/run_igie.py | 286 ++++++------ tests/run_ixrt.py | 441 +++++++++--------- 6 files changed, 391 insertions(+), 358 deletions(-) diff --git a/models/audio/speech_recognition/conformer/igie/ci/prepare.sh b/models/audio/speech_recognition/conformer/igie/ci/prepare.sh index 49ec77a0..5553e220 100644 --- a/models/audio/speech_recognition/conformer/igie/ci/prepare.sh +++ b/models/audio/speech_recognition/conformer/igie/ci/prepare.sh @@ -28,6 +28,7 @@ pip3 install -r requirements.txt ln -s /mnt/deepspark/data/checkpoints/conformer_checkpoints.tar ./ tar xf conformer_checkpoints.tar -cp /mnt/deepspark/data/datasets/aishell_test_data.tar ./ -tar xf aishell_test_data.tar -bash scripts/aishell_data_prepare.sh ./aishell_test_data ./tools \ No newline at end of file +ln -s /mnt/deepspark/data/datasets/aishell_test_data ./aishell_test_data +# cp /mnt/deepspark/data/datasets/aishell_test_data.tar ./ +# tar xf aishell_test_data.tar +# bash scripts/aishell_data_prepare.sh ./aishell_test_data ./tools \ No newline at end of file diff --git a/models/audio/speech_recognition/conformer/ixrt/ci/prepare.sh b/models/audio/speech_recognition/conformer/ixrt/ci/prepare.sh index 40fd32af..75d38ed3 100644 --- a/models/audio/speech_recognition/conformer/ixrt/ci/prepare.sh +++ b/models/audio/speech_recognition/conformer/ixrt/ci/prepare.sh @@ -29,6 +29,7 @@ pip3 install -r requirements.txt ln -s /root/data/checkpoints/conformer_checkpoints.tar ./ tar xf conformer_checkpoints.tar -cp /root/data/datasets/aishell_test_data.tar ./ -tar xf aishell_test_data.tar -bash scripts/aishell_data_prepare.sh ./aishell_test_data ./tools \ No newline at end of file +ln -s /root/data/datasets/aishell_test_data ./aishell_test_data +# cp /root/data/datasets/aishell_test_data.tar ./ +# tar xf aishell_test_data.tar +# bash scripts/aishell_data_prepare.sh ./aishell_test_data ./tools \ No newline at end of file diff --git a/models/audio/speech_recognition/conformer/ixrt/scripts/infer_conformer_fp16_accuracy.sh b/models/audio/speech_recognition/conformer/ixrt/scripts/infer_conformer_fp16_accuracy.sh index 5a1abbe2..e0e58baf 100644 --- a/models/audio/speech_recognition/conformer/ixrt/scripts/infer_conformer_fp16_accuracy.sh +++ b/models/audio/speech_recognition/conformer/ixrt/scripts/infer_conformer_fp16_accuracy.sh @@ -51,11 +51,11 @@ cd ${PROJECT_DIR} python3 build_engine.py \ --onnx_model ${MODEL_DIR}/conformer_fp16_trt.onnx \ - --engine ${MODEL_DIR}/conformer_fp16_trt.engine "$@" ;check_status + --engine ${MODEL_DIR}/conformer_fp16_trt.engine;check_status python3 ixrt_inference_accuracy.py \ --infer_type fp16 \ --batch_size ${batchsize} \ --data_dir ${DATA_DIR} \ - --model_dir ${MODEL_DIR} "$@"; check_status + --model_dir ${MODEL_DIR}; check_status exit ${EXIT_STATUS} \ No newline at end of file diff --git a/models/audio/speech_recognition/conformer/ixrt/scripts/infer_conformer_fp16_performance.sh b/models/audio/speech_recognition/conformer/ixrt/scripts/infer_conformer_fp16_performance.sh index b62a2aa8..025b9a12 100644 --- a/models/audio/speech_recognition/conformer/ixrt/scripts/infer_conformer_fp16_performance.sh +++ b/models/audio/speech_recognition/conformer/ixrt/scripts/infer_conformer_fp16_performance.sh @@ -52,11 +52,11 @@ cd ${PROJECT_DIR} python3 build_engine.py \ --onnx_model ${MODEL_DIR}/conformer_fp16_trt.onnx \ - --engine ${MODEL_DIR}/conformer_fp16_trt.engine "$@" ;check_status + --engine ${MODEL_DIR}/conformer_fp16_trt.engine;check_status python3 ixrt_inference_performance.py \ --infer_type fp16 \ --batch_size ${batchsize} \ --data_dir ${DATA_DIR} \ - --model_dir ${MODEL_DIR} "$@"; check_status + --model_dir ${MODEL_DIR}; check_status exit ${EXIT_STATUS} \ No newline at end of file diff --git a/tests/run_igie.py b/tests/run_igie.py index 65b3b1a7..63841081 100644 --- a/tests/run_igie.py +++ b/tests/run_igie.py @@ -48,7 +48,7 @@ def main(): if not test_model: logging.error("test model case is empty") sys.exit(-1) - + batch_size = os.environ.get("BS_LISTS") model = get_model_config(test_model) if not model: logging.error("mode config is empty") @@ -63,7 +63,7 @@ def main(): logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") d_url = model["download_url"] if d_url is not None: - result = run_clf_testcase(model) + result = run_clf_testcase(model, batch_size) check_model_result(result) logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") logging.info(f"End running {model['model_name']} test case.") @@ -73,7 +73,7 @@ def main(): logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") d_url = model["download_url"] if d_url is not None: - result = run_detec_testcase(model) + result = run_detec_testcase(model, batch_size) check_model_result(result) logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") logging.info(f"End running {model['model_name']} test case.") @@ -113,7 +113,7 @@ def main(): logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") d_url = model["download_url"] if d_url is not None: - result = run_speech_testcase(model) + result = run_speech_testcase(model, batch_size) check_model_result(result) logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") logging.info(f"End running {model['model_name']} test case.") @@ -123,7 +123,7 @@ def main(): logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") d_url = model["download_url"] if d_url is not None: - result = run_nlp_testcase(model) + result = run_nlp_testcase(model, batch_size) check_model_result(result) logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") logging.info(f"End running {model['model_name']} test case.") @@ -148,7 +148,8 @@ def check_model_result(result): break result["status"] = status -def run_clf_testcase(model): +def run_clf_testcase(model, batch_size): + batch_size_list = batch_size.split(",") if batch_size else [] model_name = model["model_name"] result = { "name": model_name, @@ -177,57 +178,60 @@ def run_clf_testcase(model): run_script(prepare_script) + if model_name == "unet": + base_script = f""" + export DATASETS_DIR=/mnt/deepspark/data/datasets/{dataset_n} + export RUN_DIR=../../igie_common/ + cd ../{model['model_path']} + """ + else: + base_script = f""" + export DATASETS_DIR=/mnt/deepspark/data/datasets/imagenet-val + export RUN_DIR=../../igie_common/ + cd ../{model['model_path']} + """ + for prec in model["precisions"]: - logging.info(f"Start running {model_name} {prec} test case") - if model_name == "unet": - script = f""" - export DATASETS_DIR=/mnt/deepspark/data/datasets/{dataset_n} - export RUN_DIR=../../igie_common/ - cd ../{model['model_path']} - bash scripts/infer_{model_name}_{prec}_accuracy.sh - bash scripts/infer_{model_name}_{prec}_performance.sh - """ - else: - script = f""" - export DATASETS_DIR=/mnt/deepspark/data/datasets/imagenet-val - export RUN_DIR=../../igie_common/ - cd ../{model['model_path']} - bash scripts/infer_{model_name}_{prec}_accuracy.sh - bash scripts/infer_{model_name}_{prec}_performance.sh + result["result"].setdefault(prec, {"status": "FAIL"}) + for bs in batch_size_list: + result["result"][prec].setdefault(bs, {}) + logging.info(f"Start running {model_name} {prec} bs={bs} test case") + script = base_script + f""" + bash scripts/infer_{model_name}_{prec}_accuracy.sh --bs {bs} + bash scripts/infer_{model_name}_{prec}_performance.sh --bs {bs} """ - r, t = run_script(script) - sout = r.stdout - pattern = r"\* ([\w\d ]+):\s*([\d.]+)[ ms%]*, ([\w\d ]+):\s*([\d.]+)[ ms%]*" - matchs = re.findall(pattern, sout) - for m in matchs: - result["result"].setdefault(prec, {"status": "FAIL"}) - try: - result["result"][prec] = result["result"][prec] | {m[0]: float(m[1]), m[2]: float(m[3])} - except ValueError: - print("The string cannot be converted to a float.") - result["result"][prec] = result["result"][prec] | {m[0]: m[1], m[2]: m[3]} - if matchs: - if len(matchs) == 2: - result["result"][prec]["status"] = "PASS" - else: - # Define regex pattern to match key-value pairs inside curly braces - kv_pattern = r"'(\w+)'\s*:\s*([\d.]+)" - # Find all matches - kv_matches = re.findall(kv_pattern, sout) - for key, value in kv_matches: + r, t = run_script(script) + sout = r.stdout + pattern = r"\* ([\w\d ]+):\s*([\d.]+)[ ms%]*, ([\w\d ]+):\s*([\d.]+)[ ms%]*" + matchs = re.findall(pattern, sout) + for m in matchs: + try: + result["result"][prec][bs] = result["result"][prec][bs] | {m[0]: float(m[1]), m[2]: float(m[3])} + except ValueError: + print("The string cannot be converted to a float.") + result["result"][prec][bs] = result["result"][prec][bs] | {m[0]: m[1], m[2]: m[3]} + if matchs: + if len(matchs) == 2: result["result"][prec]["status"] = "PASS" - try: - result["result"][prec][key] = float(value) - except ValueError: - print("The string cannot be converted to a float.") - result["result"][prec][key] = value - - result["result"][prec]["Cost time (s)"] = t - logging.debug(f"matchs:\n{matchs}") + else: + # Define regex pattern to match key-value pairs inside curly braces + kv_pattern = r"'(\w+)'\s*:\s*([\d.]+)" + # Find all matches + kv_matches = re.findall(kv_pattern, sout) + for key, value in kv_matches: + result["result"][prec]["status"] = "PASS" + try: + result["result"][prec][bs][key] = float(value) + except ValueError: + print("The string cannot be converted to a float.") + result["result"][prec][bs][key] = value + result["result"][prec][bs]["Cost time (s)"] = t + logging.debug(f"matchs:\n{matchs}") return result -def run_detec_testcase(model): +def run_detec_testcase(model, batch_size): + batch_size_list = batch_size.split(",") if batch_size else [] model_name = model["model_name"] result = { "name": model_name, @@ -258,46 +262,49 @@ def run_detec_testcase(model): run_script(prepare_script) - for prec in model["precisions"]: - logging.info(f"Start running {model_name} {prec} test case") - script = f""" + base_script = f""" cd ../{model['model_path']} export DATASETS_DIR=./{dataset_n}/ - bash scripts/infer_{model_name}_{prec}_accuracy.sh - bash scripts/infer_{model_name}_{prec}_performance.sh - """ + """ - r, t = run_script(script) - sout = r.stdout - pattern = r"\* ([\w\d ]+):\s*([\d.]+)[ ms%]*, ([\w\d ]+):\s*([\d.]+)[ ms%]*" - matchs = re.findall(pattern, sout) - for m in matchs: - result["result"].setdefault(prec, {"status": "FAIL"}) - try: - result["result"][prec] = result["result"][prec] | {m[0]: float(m[1]), m[2]: float(m[3])} - except ValueError: - print("The string cannot be converted to a float.") - result["result"][prec] = result["result"][prec] | {m[0]: m[1], m[2]: m[3]} - pattern = r"Average Precision \(AP\) @\[ (IoU=0.50[:\d.]*)\s*\| area= all \| maxDets=\s?\d+\s?\] =\s*([\d.]+)" - matchs = re.findall(pattern, sout) - for m in matchs: - result["result"].setdefault(prec, {}) - try: - result["result"][prec] = result["result"][prec] | {m[0]: float(m[1])} - except ValueError: - print("The string cannot be converted to a float.") - result["result"][prec] = result["result"][prec] | {m[0]: m[1]} - if matchs and len(matchs) == 2: - result["result"][prec]["status"] = "PASS" - else: - pattern = METRIC_PATTERN + for prec in model["precisions"]: + result["result"].setdefault(prec, {"status": "FAIL"}) + for bs in batch_size_list: + result["result"][prec].setdefault(bs, {}) + logging.info(f"Start running {model_name} {prec} bs={bs} test case") + script = base_script + f""" + bash scripts/infer_{model_name}_{prec}_accuracy.sh --bs {bs} + bash scripts/infer_{model_name}_{prec}_performance.sh --bs {bs} + """ + + r, t = run_script(script) + sout = r.stdout + pattern = r"\* ([\w\d ]+):\s*([\d.]+)[ ms%]*, ([\w\d ]+):\s*([\d.]+)[ ms%]*" matchs = re.findall(pattern, sout) - if matchs and len(matchs) == 1: - result["result"].setdefault(prec, {}) - result["result"][prec].update(get_metric_result(matchs[0])) + for m in matchs: + try: + result["result"][prec][bs] = result["result"][prec].get(bs, {}) | {m[0]: float(m[1]), m[2]: float(m[3])} + except ValueError: + print("The string cannot be converted to a float.") + result["result"][prec][bs] = result["result"][prec].get(bs, {}) | {m[0]: m[1], m[2]: m[3]} + pattern = r"Average Precision \(AP\) @\[ (IoU=0.50[:\d.]*)\s*\| area= all \| maxDets=\s?\d+\s?\] =\s*([\d.]+)" + matchs = re.findall(pattern, sout) + for m in matchs: + try: + result["result"][prec][bs] = result["result"][prec].get(bs, {}) | {m[0]: float(m[1])} + except ValueError: + print("The string cannot be converted to a float.") + result["result"][prec][bs] = result["result"][prec].get(bs, {}) | {m[0]: m[1]} + if matchs and len(matchs) == 2: result["result"][prec]["status"] = "PASS" - result["result"][prec]["Cost time (s)"] = t - logging.debug(f"matchs:\n{matchs}") + else: + pattern = METRIC_PATTERN + matchs = re.findall(pattern, sout) + if matchs and len(matchs) == 1: + result["result"][prec][bs].update(get_metric_result(matchs[0])) + result["result"][prec]["status"] = "PASS" + result["result"][prec][bs]["Cost time (s)"] = t + logging.debug(f"matchs:\n{matchs}") return result @@ -477,7 +484,8 @@ def run_multi_object_tracking_testcase(model): return result # BERT series models -def run_nlp_testcase(model): +def run_nlp_testcase(model, batch_size): + batch_size_list = batch_size.split(",") if batch_size else [] model_name = model["model_name"] result = { "name": model_name, @@ -511,32 +519,37 @@ def run_nlp_testcase(model): run_script(prepare_script) - for prec in model["precisions"]: - logging.info(f"Start running {model_name} {prec} test case") - script = f""" + base_script = f""" set -x export DATASETS_DIR=/mnt/deepspark/data/datasets/{dataset_n} cd ../{model['model_path']} - bash scripts/infer_{model_name}_{prec}_accuracy.sh - bash scripts/infer_{model_name}_{prec}_performance.sh - """ + """ + for prec in model["precisions"]: + result["result"].setdefault(prec, {"status": "FAIL"}) + for bs in batch_size_list: + result["result"][prec].setdefault(bs, {}) + logging.info(f"Start running {model_name} {prec} bs={bs} test case") + script = base_script + f""" + bash scripts/infer_{model_name}_{prec}_accuracy.sh + bash scripts/infer_{model_name}_{prec}_performance.sh + """ - r, t = run_script(script) - sout = r.stdout + r, t = run_script(script) + sout = r.stdout - pattern = METRIC_PATTERN - matchs = re.findall(pattern, sout) - result["result"].setdefault(prec, {"status": "FAIL"}) - logging.debug(f"matchs:\n{matchs}") - for m in matchs: - result["result"][prec].update(get_metric_result(m)) - if len(matchs) == 2: - result["result"][prec]["status"] = "PASS" + pattern = METRIC_PATTERN + matchs = re.findall(pattern, sout) + logging.debug(f"matchs:\n{matchs}") + for m in matchs: + result["result"][prec][bs].update(get_metric_result(m)) + if len(matchs) == 2: + result["result"][prec]["status"] = "PASS" - result["result"][prec]["Cost time (s)"] = t + result["result"][prec][bs]["Cost time (s)"] = t return result -def run_speech_testcase(model): +def run_speech_testcase(model, batch_size): + batch_size_list = batch_size.split(",") if batch_size else [] model_name = model["model_name"] result = { "name": model_name, @@ -565,42 +578,45 @@ def run_speech_testcase(model): run_script(prepare_script) - for prec in model["precisions"]: - logging.info(f"Start running {model_name} {prec} test case") - script = f""" + base_script = f""" cd ../{model['model_path']} export PYTHONPATH=./wenet:$PYTHONPATH echo $PYTHONPATH - bash scripts/infer_{model_name}_{prec}_accuracy.sh - bash scripts/infer_{model_name}_{prec}_performance.sh - """ + """ - r, t = run_script(script) - sout = r.stdout - pattern = r"\* ([\w\d ]+):\s*([\d.]+)[ ms%]*, ([\w\d ]+):\s*([\d.]+)[ ms%]*" - matchs = re.findall(pattern, sout) - for m in matchs: - result["result"].setdefault(prec, {"status": "FAIL"}) - try: - result["result"][prec] = result["result"][prec] | {m[0]: float(m[1]), m[2]: float(m[3])} - except ValueError: - print("The string cannot be converted to a float.") - result["result"][prec] = result["result"][prec] | {m[0]: m[1], m[2]: m[3]} - pattern = METRIC_PATTERN - matchs = re.findall(pattern, sout) + for prec in model["precisions"]: result["result"].setdefault(prec, {"status": "FAIL"}) - if matchs: - if len(matchs) == 1: - result["result"].setdefault(prec, {}) - result["result"][prec].update(get_metric_result(matchs[0])) - result["result"][prec]["status"] = "PASS" - else: - for m in matchs: - result["result"][prec].update(get_metric_result(m)) - if len(matchs) == 2: + for bs in batch_size_list: + result["result"][prec].setdefault(bs, {}) + logging.info(f"Start running {model_name} {prec} bs={bs} test case") + script = base_script + f""" + bash scripts/infer_{model_name}_{prec}_accuracy.sh + bash scripts/infer_{model_name}_{prec}_performance.sh + """ + + r, t = run_script(script) + sout = r.stdout + pattern = r"\* ([\w\d ]+):\s*([\d.]+)[ ms%]*, ([\w\d ]+):\s*([\d.]+)[ ms%]*" + matchs = re.findall(pattern, sout) + for m in matchs: + try: + result["result"][prec][bs] = result["result"][prec][bs] | {m[0]: float(m[1]), m[2]: float(m[3])} + except ValueError: + print("The string cannot be converted to a float.") + result["result"][prec][bs] = result["result"][prec][bs] | {m[0]: m[1], m[2]: m[3]} + pattern = METRIC_PATTERN + matchs = re.findall(pattern, sout) + if matchs: + if len(matchs) == 1: + result["result"][prec][bs].update(get_metric_result(matchs[0])) result["result"][prec]["status"] = "PASS" - result["result"][prec]["Cost time (s)"] = t - logging.debug(f"matchs:\n{matchs}") + else: + for m in matchs: + result["result"][prec][bs].update(get_metric_result(m)) + if len(matchs) == 2: + result["result"][prec]["status"] = "PASS" + result["result"][prec][bs]["Cost time (s)"] = t + logging.debug(f"matchs:\n{matchs}") return result def get_metric_result(str): diff --git a/tests/run_ixrt.py b/tests/run_ixrt.py index adf06ad2..c924124b 100644 --- a/tests/run_ixrt.py +++ b/tests/run_ixrt.py @@ -48,7 +48,7 @@ def main(): if not test_model: logging.error("test model case is empty") sys.exit(-1) - + batch_size = os.environ.get("BS_LISTS") model = get_model_config(test_model) if not model: logging.error("mode config is empty") @@ -63,7 +63,7 @@ def main(): logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") d_url = model["download_url"] if d_url is not None: - result = run_clf_testcase(model) + result = run_clf_testcase(model, batch_size) check_model_result(result) logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") logging.info(f"End running {model['model_name']} test case.") @@ -73,7 +73,7 @@ def main(): logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") d_url = model["download_url"] if d_url is not None: - result = run_detec_testcase(model) + result = run_detec_testcase(model, batch_size) check_model_result(result) logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") logging.info(f"End running {model['model_name']} test case.") @@ -93,7 +93,7 @@ def main(): logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") d_url = model["download_url"] if d_url is not None: - result = run_speech_testcase(model) + result = run_speech_testcase(model, batch_size) check_model_result(result) logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") logging.info(f"End running {model['model_name']} test case.") @@ -113,7 +113,7 @@ def main(): logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") d_url = model["download_url"] if d_url is not None: - result = run_nlp_testcase(model) + result = run_nlp_testcase(model, batch_size) check_model_result(result) logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") logging.info(f"End running {model['model_name']} test case.") @@ -148,7 +148,8 @@ def check_model_result(result): break result["status"] = status -def run_clf_testcase(model): +def run_clf_testcase(model, batch_size): + batch_size_list = batch_size.split(",") if batch_size else [] model_name = model["model_name"] result = { "name": model_name, @@ -185,11 +186,8 @@ def run_clf_testcase(model): } combined_pattern = re.compile("|".join(f"(?P<{name}>{pattern})" for name, pattern in patterns.items())) - - for prec in model["precisions"]: - logging.info(f"Start running {model_name} {prec} test case") - if model_name == "clip": - script = f""" + if model_name == "clip": + base_script = f""" cd ../{model['model_path']} export DATASETS_DIR=/root/data/datasets/imagenet-val export OPTIMIER_FILE=./iluvatar-corex-ixrt/tools/optimizer/optimizer.py @@ -197,23 +195,19 @@ def run_clf_testcase(model): export CHECKPOINTS_DIR=./checkpoints/clip export RUN_DIR=../../ixrt_common/ export CONFIG_DIR=../../ixrt_common/config/{config_name}_CONFIG - bash scripts/infer_{model_name}_{prec}_accuracy.sh - bash scripts/infer_{model_name}_{prec}_performance.sh - """ - else: - script = f""" + """ + else: + base_script = f""" cd ../{model['model_path']} export DATASETS_DIR=/root/data/datasets/imagenet-val export PROJ_DIR=../../ixrt_common/ export CHECKPOINTS_DIR=./checkpoints export RUN_DIR=../../ixrt_common/ export CONFIG_DIR=../../ixrt_common/config/{config_name}_CONFIG - bash scripts/infer_{model_name}_{prec}_accuracy.sh - bash scripts/infer_{model_name}_{prec}_performance.sh - """ + """ - if model_name == "swin_transformer_large": - script = f""" + if model_name == "swin_transformer_large": + base_script = f""" cd ../{model['model_path']} export ORIGIN_ONNX_NAME=./swin-large-torch-fp32 export OPTIMIER_FILE=./iluvatar-corex-ixrt/tools/optimizer/optimizer.py @@ -221,47 +215,61 @@ def run_clf_testcase(model): bash scripts/infer_swinl_fp16_performance.sh cd ./ByteMLPerf/byte_infer_perf/general_perf python3 core/perf_engine.py --hardware_type ILUVATAR --task swin-large-torch-fp32 + """ + for prec in model["precisions"]: + result["result"].setdefault(prec, {"status": "FAIL"}) + for bs in batch_size_list: + result["result"][prec].setdefault(bs, {}) + logging.info(f"Start running {model_name} {prec} bs={bs} test case") + + script = base_script + f""" + bash scripts/infer_{model_name}_{prec}_accuracy.sh --bs {bs} + bash scripts/infer_{model_name}_{prec}_performance.sh --bs {bs} """ - r, t = run_script(script) - sout = r.stdout - matchs = combined_pattern.finditer(sout) - result["result"].setdefault(prec, {"status": "FAIL"}) - match_count = 0 - for match in matchs: - for name, value in match.groupdict().items(): - if value: - match_count += 1 - result["result"][prec][name] = float(f"{float(value.split(':')[1].strip()):.3f}") - break - if match_count == len(patterns): - result["result"][prec]["status"] = "PASS" + if model_name == "swin_transformer_large": + script = base_script - if model_name == "swin_transformer_large": - pattern = r'Throughput: (\d+\.\d+) qps' - matchs = re.findall(pattern, sout) - for m in matchs: + r, t = run_script(script) + sout = r.stdout + matchs = combined_pattern.finditer(sout) + match_count = 0 + for match in matchs: + for name, value in match.groupdict().items(): + logging.debug(f"matchs: name: {name}, value: {value}") + if value: + match_count += 1 + result["result"][prec][bs][name] = float(f"{float(value.split(':')[1].strip()):.3f}") + break + if match_count == len(patterns): + result["result"][prec]["status"] = "PASS" + + if model_name == "swin_transformer_large": + pattern = r'Throughput: (\d+\.\d+) qps' + matchs = re.findall(pattern, sout) + for m in matchs: + result["result"].setdefault(prec, {"status": "FAIL"}) + try: + result["result"][prec][bs]["QPS"] = float(m) + except ValueError: + print("The string cannot be converted to a float.") + result["result"][prec][bs]["QPS"] = m + + pattern = METRIC_PATTERN + matchs = re.findall(pattern, sout) result["result"].setdefault(prec, {"status": "FAIL"}) - try: - result["result"][prec]["QPS"] = float(m) - except ValueError: - print("The string cannot be converted to a float.") - result["result"][prec]["QPS"] = m + logging.debug(f"matchs:\n{matchs}") + for m in matchs: + result["result"][prec][bs].update(get_metric_result(m)) + if len(matchs) == 1: + result["result"][prec]["status"] = "PASS" - pattern = METRIC_PATTERN - matchs = re.findall(pattern, sout) - result["result"].setdefault(prec, {"status": "FAIL"}) + result["result"][prec][bs]["Cost time (s)"] = t logging.debug(f"matchs:\n{matchs}") - for m in matchs: - result["result"][prec].update(get_metric_result(m)) - if len(matchs) == 1: - result["result"][prec]["status"] = "PASS" - - result["result"][prec]["Cost time (s)"] = t - logging.debug(f"matchs:\n{matchs}") return result -def run_detec_testcase(model): +def run_detec_testcase(model, batch_size): + batch_size_list = batch_size.split(",") if batch_size else [] model_name = model["model_name"] result = { "name": model_name, @@ -289,78 +297,81 @@ def run_detec_testcase(model): if model_name == "yolov5": config_name = "YOLOV5M" + if model_name in ["yolov3", "yolov5", "yolov5s", "yolov7", "atss", "paa", "retinanet", "yolof", "fcos"]: + base_script = f""" + cd ../{model['model_path']} + export DATASETS_DIR=./{dataset_n}/ + export MODEL_PATH=./{model_name}.onnx + export PROJ_DIR=./ + export CHECKPOINTS_DIR=./checkpoints + export COCO_GT=./{dataset_n}/annotations/instances_val2017.json + export EVAL_DIR=./{dataset_n}/images/val2017 + export RUN_DIR=../../ixrt_common + export CONFIG_DIR=../../ixrt_common/config/{config_name}_CONFIG + """ + else: + base_script = f""" + cd ../{model['model_path']} + export DATASETS_DIR=./{dataset_n}/ + export MODEL_PATH=./{model_name}.onnx + export PROJ_DIR=./ + export CHECKPOINTS_DIR=./checkpoints + export COCO_GT=./{dataset_n}/annotations/instances_val2017.json + export EVAL_DIR=./{dataset_n}/images/val2017 + export RUN_DIR=./ + export CONFIG_DIR=config/{config_name}_CONFIG + """ + for prec in model["precisions"]: - logging.info(f"Start running {model_name} {prec} test case") - result["result"].setdefault(prec, {}) result["result"].setdefault(prec, {"status": "FAIL"}) - if model_name in ["yolov3", "yolov5", "yolov5s", "yolov7", "atss", "paa", "retinanet", "yolof", "fcos"]: - script = f""" - cd ../{model['model_path']} - export DATASETS_DIR=./{dataset_n}/ - export MODEL_PATH=./{model_name}.onnx - export PROJ_DIR=./ - export CHECKPOINTS_DIR=./checkpoints - export COCO_GT=./{dataset_n}/annotations/instances_val2017.json - export EVAL_DIR=./{dataset_n}/images/val2017 - export RUN_DIR=../../ixrt_common - export CONFIG_DIR=../../ixrt_common/config/{config_name}_CONFIG - bash scripts/infer_{model_name}_{prec}_accuracy.sh - bash scripts/infer_{model_name}_{prec}_performance.sh - """ - else: - script = f""" - cd ../{model['model_path']} - export DATASETS_DIR=./{dataset_n}/ - export MODEL_PATH=./{model_name}.onnx - export PROJ_DIR=./ - export CHECKPOINTS_DIR=./checkpoints - export COCO_GT=./{dataset_n}/annotations/instances_val2017.json - export EVAL_DIR=./{dataset_n}/images/val2017 - export RUN_DIR=./ - export CONFIG_DIR=config/{config_name}_CONFIG - bash scripts/infer_{model_name}_{prec}_accuracy.sh - bash scripts/infer_{model_name}_{prec}_performance.sh + for bs in batch_size_list: + result["result"][prec].setdefault(bs, {}) + logging.info(f"Start running {model_name} {prec} bs={bs} test case") + result["result"].setdefault(prec, {"status": "FAIL"}) + script = base_script + f""" + bash scripts/infer_{model_name}_{prec}_accuracy.sh --bs {bs} + bash scripts/infer_{model_name}_{prec}_performance.sh --bs {bs} """ - if model_name == "rtmpose": - script = f""" - cd ../{model['model_path']} - python3 predict.py --model data/rtmpose/rtmpose_opt.onnx --precision fp16 --img_path demo/demo.jpg - """ - - r, t = run_script(script) - sout = r.stdout - fps_pattern = r"(?PFPS\s*:\s*(\d+\.?\d*))" - e2e_pattern = r"(?P\s*E2E time\s*:\s*(\d+\.\d+)\s)" - combined_pattern = re.compile(f"{fps_pattern}|{e2e_pattern}") - matchs = combined_pattern.finditer(sout) - for match in matchs: - for name, value in match.groupdict().items(): - if value: - try: - result["result"][prec][name] = float(f"{float(value.split(':')[1].strip()):.3f}") - break - except ValueError: - print("The string cannot be converted to a float.") - result["result"][prec][name] = value - pattern = r"Average Precision \(AP\) @\[ (IoU=0.50[:\d.]*)\s*\| area= all \| maxDets=\s?\d+\s?\] =\s*([\d.]+)" - matchs = re.findall(pattern, sout) - for m in matchs: - try: - result["result"][prec][m[0]] = float(m[1]) - except ValueError: - print("The string cannot be converted to a float.") - result["result"][prec][m[0]] = m[1] - if matchs and len(matchs) == 2: - result["result"][prec]["status"] = "PASS" - else: - pattern = METRIC_PATTERN + if model_name == "rtmpose": + script = f""" + cd ../{model['model_path']} + python3 predict.py --model data/rtmpose/rtmpose_opt.onnx --precision fp16 --img_path demo/demo.jpg + """ + + r, t = run_script(script) + sout = r.stdout + fps_pattern = r"(?PFPS\s*:\s*(\d+\.?\d*))" + e2e_pattern = r"(?P\s*E2E time\s*:\s*(\d+\.\d+)\s)" + combined_pattern = re.compile(f"{fps_pattern}|{e2e_pattern}") + matchs = combined_pattern.finditer(sout) + for match in matchs: + for name, value in match.groupdict().items(): + if value: + try: + result["result"][prec][bs][name] = float(f"{float(value.split(':')[1].strip()):.3f}") + break + except ValueError: + print("The string cannot be converted to a float.") + result["result"][prec][bs][name] = value + pattern = r"Average Precision \(AP\) @\[ (IoU=0.50[:\d.]*)\s*\| area= all \| maxDets=\s?\d+\s?\] =\s*([\d.]+)" matchs = re.findall(pattern, sout) - if matchs and len(matchs) == 1: - result["result"][prec].update(get_metric_result(matchs[0])) + for m in matchs: + try: + result["result"][prec][bs][m[0]] = float(m[1]) + except ValueError: + print("The string cannot be converted to a float.") + result["result"][prec][bs][m[0]] = m[1] + if matchs and len(matchs) == 2: result["result"][prec]["status"] = "PASS" - result["result"][prec]["Cost time (s)"] = t - logging.debug(f"matchs:\n{matchs}") + else: + pattern = METRIC_PATTERN + matchs = re.findall(pattern, sout) + if matchs and len(matchs) == 1: + result["result"][prec][bs].update(get_metric_result(matchs[0])) + result["result"][prec]["status"] = "PASS" + result["result"][prec][bs]["Cost time (s)"] = t + logging.debug(f"matchs:\n{matchs}") return result @@ -496,7 +507,8 @@ def run_multi_object_tracking_testcase(model): return result # BERT series models -def run_nlp_testcase(model): +def run_nlp_testcase(model, batch_size): + batch_size_list = batch_size.split(",") if batch_size else [] model_name = model["model_name"] result = { "name": model_name, @@ -528,97 +540,99 @@ def run_nlp_testcase(model): run_script(prepare_script) - for prec in model["precisions"]: - logging.info(f"Start running {model_name} {prec} test case") - script = f""" + base_script = f""" set -x cd ../{model['model_path']} export ORIGIN_ONNX_NAME=./data/open_{model_name}/{model_name} export OPTIMIER_FILE=./iluvatar-corex-ixrt/tools/optimizer/optimizer.py export PROJ_PATH=./ - bash scripts/infer_{model_name}_{prec}_performance.sh + bash scripts/infer_{model_name}_fp16_performance.sh cd ./ByteMLPerf/byte_infer_perf/general_perf + """ + if model_name == "roformer" or model_name == "wide_and_deep": + if model_name == "wide_and_deep": + model_name = "widedeep" + base_script += f""" + python3 core/perf_engine.py --hardware_type ILUVATAR --task {model_name}-tf-fp32 + """ + elif model_name == "videobert": + base_script += f""" + python3 core/perf_engine.py --hardware_type ILUVATAR --task {model_name}-onnx-fp32 + """ + else: + # model_name == "roberta" or model_name == "deberta" or model_name == "albert" + base_script += f""" + python3 core/perf_engine.py --hardware_type ILUVATAR --task {model_name}-torch-fp32 """ - if model_name == "roformer" or model_name == "wide_and_deep": - if model_name == "wide_and_deep": - model_name = "widedeep" - script += f""" - python3 core/perf_engine.py --hardware_type ILUVATAR --task {model_name}-tf-fp32 - """ - elif model_name == "videobert": - script += f""" - python3 core/perf_engine.py --hardware_type ILUVATAR --task {model_name}-onnx-fp32 - """ - else: - # model_name == "roberta" or model_name == "deberta" or model_name == "albert" - script += f""" - python3 core/perf_engine.py --hardware_type ILUVATAR --task {model_name}-torch-fp32 - """ + for prec in model["precisions"]: + result["result"].setdefault(prec, {"status": "FAIL"}) + for bs in batch_size_list: + result["result"][prec].setdefault(bs, {}) + logging.info(f"Start running {model_name} {prec} bs: {bs} test case") + script = base_script - if model_name == "bert_base_squad": - script = f""" - set -x - cd ../{model['model_path']}/python - bash script/infer_{model_name}_{prec}_ixrt.sh - """ - elif model_name == "bert_large_squad": - script = f""" - set -x - cd ../{model['model_path']}/ - bash script/infer_bert_large_squad_fp16_accuracy.sh - bash script/infer_bert_large_squad_fp16_performance.sh - """ - if prec == "int8": + if model_name == "bert_base_squad": + script = f""" + set -x + cd ../{model['model_path']}/python + bash script/infer_bert_base_squad_{prec}_ixrt.sh --bs {bs} + """ + elif model_name == "bert_large_squad": script = f""" set -x cd ../{model['model_path']}/ - bash script/infer_bert_large_squad_int8_accuracy.sh - bash script/infer_bert_large_squad_int8_performance.sh + bash script/infer_bert_large_squad_fp16_accuracy.sh --bs {bs} + bash script/infer_bert_large_squad_fp16_performance.sh --bs {bs} """ + if prec == "int8": + script = f""" + set -x + cd ../{model['model_path']}/ + bash script/infer_bert_large_squad_int8_accuracy.sh --bs {bs} + bash script/infer_bert_large_squad_int8_performance.sh --bs {bs} + """ - r, t = run_script(script) - sout = r.stdout - - pattern = r'Throughput: (\d+\.\d+) qps' - matchs = re.findall(pattern, sout) - for m in matchs: - result["result"].setdefault(prec, {"status": "FAIL"}) - try: - result["result"][prec]["QPS"] = float(m) - except ValueError: - print("The string cannot be converted to a float.") - result["result"][prec]["QPS"] = m + r, t = run_script(script) + sout = r.stdout - pattern = METRIC_PATTERN - matchs = re.findall(pattern, sout) - result["result"].setdefault(prec, {"status": "FAIL"}) - for m in matchs: - result["result"][prec].update(get_metric_result(m)) - result["result"][prec]["status"] = "PASS" - - if model_name == "bert_large_squad": - patterns = { - "LatencyQPS": r"Latency QPS\s*:\s*(\d+\.?\d*)", - "exact_match": r"\"exact_match\"\s*:\s*(\d+\.?\d*)", - "f1": r"\"f1\"\s*:\s*(\d+\.?\d*)" - } + pattern = r'Throughput: (\d+\.\d+) qps' + matchs = re.findall(pattern, sout) + for m in matchs: + try: + result["result"][prec][bs]["QPS"] = float(m) + except ValueError: + print("The string cannot be converted to a float.") + result["result"][prec][bs]["QPS"] = m - combined_pattern = re.compile("|".join(f"(?P<{name}>{pattern})" for name, pattern in patterns.items())) - matchs = combined_pattern.finditer(sout) - result["result"].setdefault(prec, {"status": "FAIL"}) - for match in matchs: + pattern = METRIC_PATTERN + matchs = re.findall(pattern, sout) + for m in matchs: + result["result"][prec][bs].update(get_metric_result(m)) result["result"][prec]["status"] = "PASS" - for name, value in match.groupdict().items(): - if value: - result["result"][prec][name] = float(f"{float(value.split(':')[1].strip()):.3f}") - break + + if model_name == "bert_large_squad": + patterns = { + "LatencyQPS": r"Latency QPS\s*:\s*(\d+\.?\d*)", + "exact_match": r"\"exact_match\"\s*:\s*(\d+\.?\d*)", + "f1": r"\"f1\"\s*:\s*(\d+\.?\d*)" + } + + combined_pattern = re.compile("|".join(f"(?P<{name}>{pattern})" for name, pattern in patterns.items())) + matchs = combined_pattern.finditer(sout) + for match in matchs: + result["result"][prec]["status"] = "PASS" + for name, value in match.groupdict().items(): + if value: + result["result"][prec][bs][name] = float(f"{float(value.split(':')[1].strip()):.3f}") + break - logging.debug(f"matchs:\n{matchs}") - result["result"][prec]["Cost time (s)"] = t + logging.debug(f"matchs:\n{matchs}") + result["result"][prec][bs]["Cost time (s)"] = t return result -def run_speech_testcase(model): +def run_speech_testcase(model, batch_size): + batch_size_list = batch_size.split(",") if batch_size else [] model_name = model["model_name"] result = { "name": model_name, @@ -642,32 +656,33 @@ def run_speech_testcase(model): run_script(prepare_script) for prec in model["precisions"]: - logging.info(f"Start running {model_name} {prec} test case") - script = f""" - cd ../{model['model_path']} - bash scripts/infer_{model_name}_{prec}_accuracy.sh - bash scripts/infer_{model_name}_{prec}_performance.sh - """ - - if model_name == "transformer_asr": + result["result"].setdefault(prec, {"status": "FAIL"}) + for bs in batch_size_list: + result["result"][prec].setdefault(bs, {}) + logging.info(f"Start running {model_name} {prec} bs:{bs} test case") script = f""" cd ../{model['model_path']} - python3 inference.py hparams/train_ASR_transformer.yaml --data_folder=/home/data/speechbrain/aishell --engine_path transformer.engine + bash scripts/infer_{model_name}_{prec}_accuracy.sh --bs {bs} + bash scripts/infer_{model_name}_{prec}_performance.sh --bs {bs} """ - r, t = run_script(script) - sout = r.stdout - pattern = METRIC_PATTERN - matchs = re.findall(pattern, sout) - result["result"].setdefault(prec, {"status": "FAIL"}) - logging.debug(f"matchs:\n{matchs}") - for m in matchs: - result["result"][prec].update(get_metric_result(m)) - if len(matchs) == 2: - result["result"][prec]["status"] = "PASS" + if model_name == "transformer_asr": + script = f""" + cd ../{model['model_path']} + python3 inference.py hparams/train_ASR_transformer.yaml --data_folder=/home/data/speechbrain/aishell --engine_path transformer.engine + """ - result["result"][prec]["Cost time (s)"] = t - logging.debug(f"matchs:\n{matchs}") + r, t = run_script(script) + sout = r.stdout + pattern = METRIC_PATTERN + matchs = re.findall(pattern, sout) + for m in matchs: + result["result"][prec][bs].update(get_metric_result(m)) + if len(matchs) == 2: + result["result"][prec]["status"] = "PASS" + + result["result"][prec][bs]["Cost time (s)"] = t + logging.debug(f"matchs:\n{matchs}") return result def run_instance_segmentation_testcase(model): -- Gitee