From 24ebe32f8fe9fa03b64273fae0f9c8b35220485c Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 01:04:28 +0530 Subject: [PATCH 1/6] Fix docker build script paths for subdirectory node sources --- mkconcore.py | 58 +++++++++++++++++++++++++---------------------- tests/test_cli.py | 29 ++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 27 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index f2fb6f7..090be3f 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -785,33 +785,37 @@ def cleanup_script_files(): fcopy.write('CMD ["./a.out"]\n') # 7/02/21 fbuild.write('#!/bin/bash' + "\n") - for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.rsplit(".", 1) - fbuild.write("mkdir docker-"+dockername+"\n") - fbuild.write("cd docker-"+dockername+"\n") - fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") - #copy sourcefiles from ./src into corresponding directories - fbuild.write("cp ../src/"+sourcecode+" .\n") - if langext == "py": #4/29/21 - fbuild.write("cp ../src/concore.py .\n") - elif langext == "cpp": #6/22/21 - fbuild.write("cp ../src/concore.hpp .\n") - elif langext == "v": #6/25/21 - fbuild.write("cp ../src/concore.v .\n") - if langext == "m": - fbuild.write("cp ../src/concore_*.m .\n") - fbuild.write("cp ../src/import_concore.m .\n") - if langext == "sh": #5/27/21 - fbuild.write("chmod u+x "+sourcecode+"\n") - fbuild.write("cp ../src/"+dockername+".iport concore.iport\n") - fbuild.write("cp ../src/"+dockername+".oport concore.oport\n") - #include data files in here if they exist - if os.path.isdir(sourcedir+"/"+dockername+".dir"): - fbuild.write("cp -r ../src/"+dockername+".dir/* .\n") - fbuild.write(DOCKEREXE+" build -t docker-"+dockername+" .\n") - fbuild.write("cd ..\n") + for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 + dockername,langext = sourcecode.rsplit(".", 1) + # dockername can include subdirectories (e.g. subdir/script). + # Compute a stable prefix to reach study root from docker build dir. + docker_subpath_depth = dockername.count("/") + dockername.count("\\") + docker_rel_prefix = "../" * (docker_subpath_depth + 1) + fbuild.write("mkdir docker-"+dockername+"\n") + fbuild.write("cd docker-"+dockername+"\n") + fbuild.write("cp "+docker_rel_prefix+"src/Dockerfile."+dockername+" Dockerfile\n") + #copy sourcefiles from ./src into corresponding directories + fbuild.write("cp "+docker_rel_prefix+"src/"+sourcecode+" .\n") + if langext == "py": #4/29/21 + fbuild.write("cp "+docker_rel_prefix+"src/concore.py .\n") + elif langext == "cpp": #6/22/21 + fbuild.write("cp "+docker_rel_prefix+"src/concore.hpp .\n") + elif langext == "v": #6/25/21 + fbuild.write("cp "+docker_rel_prefix+"src/concore.v .\n") + if langext == "m": + fbuild.write("cp "+docker_rel_prefix+"src/concore_*.m .\n") + fbuild.write("cp "+docker_rel_prefix+"src/import_concore.m .\n") + if langext == "sh": #5/27/21 + fbuild.write("chmod u+x "+sourcecode+"\n") + fbuild.write("cp "+docker_rel_prefix+"src/"+dockername+".iport concore.iport\n") + fbuild.write("cp "+docker_rel_prefix+"src/"+dockername+".oport concore.oport\n") + #include data files in here if they exist + if os.path.isdir(sourcedir+"/"+dockername+".dir"): + fbuild.write("cp -r "+docker_rel_prefix+"src/"+dockername+".dir/* .\n") + fbuild.write(DOCKEREXE+" build -t docker-"+dockername+" .\n") + fbuild.write("cd "+docker_rel_prefix+"\n") fbuild.close() diff --git a/tests/test_cli.py b/tests/test_cli.py index 4723002..0871fde 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -153,6 +153,35 @@ def test_run_command_subdir_source(self): self.assertEqual(result.exit_code, 0) self.assertTrue(Path('out/src/subdir/script.py').exists()) + def test_run_command_docker_subdir_source_build_paths(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + subdir = Path('test-project/src/subdir') + subdir.mkdir(parents=True, exist_ok=True) + shutil.move('test-project/src/script.py', subdir / 'script.py') + + workflow_path = Path('test-project/workflow.graphml') + content = workflow_path.read_text() + content = content.replace('N1:script.py', 'N1:subdir/script.py') + workflow_path.write_text(content) + + result = self.runner.invoke(cli, [ + 'run', + 'test-project/workflow.graphml', + '--source', 'test-project/src', + '--output', 'out', + '--type', 'docker' + ]) + self.assertEqual(result.exit_code, 0) + + build_script = Path('out/build').read_text() + self.assertIn('cp ../../src/Dockerfile.subdir/script Dockerfile', build_script) + self.assertIn('cp ../../src/subdir/script.py .', build_script) + self.assertIn('cp ../../src/subdir/script.iport concore.iport', build_script) + self.assertIn('cd ../../', build_script) + def test_run_command_shared_source_specialization_merges_edge_params(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): Path('src').mkdir() From 0ace53aa350b60a6610964c8e1d79b1fa9312260 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 23:40:03 +0530 Subject: [PATCH 2/6] Refine docker subdir fix to minimal mkconcore line changes --- mkconcore.py | 31 ++++++++++++++----------------- tests/test_cli.py | 9 +++++---- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 090be3f..0387e0c 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -789,33 +789,30 @@ def cleanup_script_files(): containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 dockername,langext = sourcecode.rsplit(".", 1) - # dockername can include subdirectories (e.g. subdir/script). - # Compute a stable prefix to reach study root from docker build dir. - docker_subpath_depth = dockername.count("/") + dockername.count("\\") - docker_rel_prefix = "../" * (docker_subpath_depth + 1) - fbuild.write("mkdir docker-"+dockername+"\n") - fbuild.write("cd docker-"+dockername+"\n") - fbuild.write("cp "+docker_rel_prefix+"src/Dockerfile."+dockername+" Dockerfile\n") + dockerbuilddir = "docker-"+dockername.replace("/", "__").replace("\\", "__") + fbuild.write("mkdir "+dockerbuilddir+"\n") + fbuild.write("cd "+dockerbuilddir+"\n") + fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") #copy sourcefiles from ./src into corresponding directories - fbuild.write("cp "+docker_rel_prefix+"src/"+sourcecode+" .\n") + fbuild.write("cp ../src/"+sourcecode+" .\n") if langext == "py": #4/29/21 - fbuild.write("cp "+docker_rel_prefix+"src/concore.py .\n") + fbuild.write("cp ../src/concore.py .\n") elif langext == "cpp": #6/22/21 - fbuild.write("cp "+docker_rel_prefix+"src/concore.hpp .\n") + fbuild.write("cp ../src/concore.hpp .\n") elif langext == "v": #6/25/21 - fbuild.write("cp "+docker_rel_prefix+"src/concore.v .\n") + fbuild.write("cp ../src/concore.v .\n") if langext == "m": - fbuild.write("cp "+docker_rel_prefix+"src/concore_*.m .\n") - fbuild.write("cp "+docker_rel_prefix+"src/import_concore.m .\n") + fbuild.write("cp ../src/concore_*.m .\n") + fbuild.write("cp ../src/import_concore.m .\n") if langext == "sh": #5/27/21 fbuild.write("chmod u+x "+sourcecode+"\n") - fbuild.write("cp "+docker_rel_prefix+"src/"+dockername+".iport concore.iport\n") - fbuild.write("cp "+docker_rel_prefix+"src/"+dockername+".oport concore.oport\n") + fbuild.write("cp ../src/"+dockername+".iport concore.iport\n") + fbuild.write("cp ../src/"+dockername+".oport concore.oport\n") #include data files in here if they exist if os.path.isdir(sourcedir+"/"+dockername+".dir"): - fbuild.write("cp -r "+docker_rel_prefix+"src/"+dockername+".dir/* .\n") + fbuild.write("cp -r ../src/"+dockername+".dir/* .\n") fbuild.write(DOCKEREXE+" build -t docker-"+dockername+" .\n") - fbuild.write("cd "+docker_rel_prefix+"\n") + fbuild.write("cd ..\n") fbuild.close() diff --git a/tests/test_cli.py b/tests/test_cli.py index 0871fde..33316a7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -177,10 +177,11 @@ def test_run_command_docker_subdir_source_build_paths(self): self.assertEqual(result.exit_code, 0) build_script = Path('out/build').read_text() - self.assertIn('cp ../../src/Dockerfile.subdir/script Dockerfile', build_script) - self.assertIn('cp ../../src/subdir/script.py .', build_script) - self.assertIn('cp ../../src/subdir/script.iport concore.iport', build_script) - self.assertIn('cd ../../', build_script) + self.assertIn('mkdir docker-subdir__script', build_script) + self.assertIn('cp ../src/Dockerfile.subdir/script Dockerfile', build_script) + self.assertIn('cp ../src/subdir/script.py .', build_script) + self.assertIn('cp ../src/subdir/script.iport concore.iport', build_script) + self.assertIn('cd ..', build_script) def test_run_command_shared_source_specialization_merges_edge_params(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): From d6f49c228208a765fd94af8c32364c77c57b42c7 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 23:41:24 +0530 Subject: [PATCH 3/6] Minimize mkconcore diff for docker subdir build path fix --- mkconcore.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 0387e0c..af19ff5 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -792,27 +792,27 @@ def cleanup_script_files(): dockerbuilddir = "docker-"+dockername.replace("/", "__").replace("\\", "__") fbuild.write("mkdir "+dockerbuilddir+"\n") fbuild.write("cd "+dockerbuilddir+"\n") - fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") - #copy sourcefiles from ./src into corresponding directories - fbuild.write("cp ../src/"+sourcecode+" .\n") - if langext == "py": #4/29/21 - fbuild.write("cp ../src/concore.py .\n") - elif langext == "cpp": #6/22/21 - fbuild.write("cp ../src/concore.hpp .\n") - elif langext == "v": #6/25/21 - fbuild.write("cp ../src/concore.v .\n") - if langext == "m": - fbuild.write("cp ../src/concore_*.m .\n") - fbuild.write("cp ../src/import_concore.m .\n") - if langext == "sh": #5/27/21 - fbuild.write("chmod u+x "+sourcecode+"\n") - fbuild.write("cp ../src/"+dockername+".iport concore.iport\n") - fbuild.write("cp ../src/"+dockername+".oport concore.oport\n") - #include data files in here if they exist - if os.path.isdir(sourcedir+"/"+dockername+".dir"): - fbuild.write("cp -r ../src/"+dockername+".dir/* .\n") - fbuild.write(DOCKEREXE+" build -t docker-"+dockername+" .\n") - fbuild.write("cd ..\n") + fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") + #copy sourcefiles from ./src into corresponding directories + fbuild.write("cp ../src/"+sourcecode+" .\n") + if langext == "py": #4/29/21 + fbuild.write("cp ../src/concore.py .\n") + elif langext == "cpp": #6/22/21 + fbuild.write("cp ../src/concore.hpp .\n") + elif langext == "v": #6/25/21 + fbuild.write("cp ../src/concore.v .\n") + if langext == "m": + fbuild.write("cp ../src/concore_*.m .\n") + fbuild.write("cp ../src/import_concore.m .\n") + if langext == "sh": #5/27/21 + fbuild.write("chmod u+x "+sourcecode+"\n") + fbuild.write("cp ../src/"+dockername+".iport concore.iport\n") + fbuild.write("cp ../src/"+dockername+".oport concore.oport\n") + #include data files in here if they exist + if os.path.isdir(sourcedir+"/"+dockername+".dir"): + fbuild.write("cp -r ../src/"+dockername+".dir/* .\n") + fbuild.write(DOCKEREXE+" build -t docker-"+dockername+" .\n") + fbuild.write("cd ..\n") fbuild.close() From 6cdf4c92395009592228612ad11f35f140c801a1 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 23:42:59 +0530 Subject: [PATCH 4/6] Drop unrelated duplicate-label change from docker subdir PR --- mkconcore.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index af19ff5..f538721 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -309,13 +309,7 @@ def cleanup_script_files(): except (IndexError, AttributeError): logging.debug('A node with no valid properties encountered and ignored') -label_values = list(nodes_dict.values()) -duplicates = {label for label in label_values if label_values.count(label) > 1} -if duplicates: - logging.error(f"Duplicate node labels found: {sorted(duplicates)}") - quit() - -for edge in edges_text: +for edge in edges_text: try: data = edge.find('data', recursive=False) if data: From a5d58e8b0a49bf89e4b0c45d1308bb7e9f82095e Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 23:44:59 +0530 Subject: [PATCH 5/6] Restore duplicate-label guard while keeping docker subdir fix scoped --- mkconcore.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index f538721..2caccca 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -284,7 +284,7 @@ def cleanup_script_files(): nodes_dict = dict() node_id_to_label_map = dict() # Helper to get clean node labels from GraphML ID -for node in nodes_text: +for node in nodes_text: try: data = node.find('data', recursive=False) if data: @@ -306,9 +306,15 @@ def cleanup_script_files(): nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] - except (IndexError, AttributeError): - logging.debug('A node with no valid properties encountered and ignored') - + except (IndexError, AttributeError): + logging.debug('A node with no valid properties encountered and ignored') + +label_values = list(nodes_dict.values()) +duplicates = {label for label in label_values if label_values.count(label) > 1} +if duplicates: + logging.error(f"Duplicate node labels found: {sorted(duplicates)}") + quit() + for edge in edges_text: try: data = edge.find('data', recursive=False) From 7a1f0e2d8f3cf8dcf390acc1fd29ce0b854c75e6 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 23:46:19 +0530 Subject: [PATCH 6/6] Reduce mkconcore changes to only docker build-dir normalization --- mkconcore.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 2caccca..5b0286a 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -284,7 +284,7 @@ def cleanup_script_files(): nodes_dict = dict() node_id_to_label_map = dict() # Helper to get clean node labels from GraphML ID -for node in nodes_text: +for node in nodes_text: try: data = node.find('data', recursive=False) if data: @@ -306,16 +306,16 @@ def cleanup_script_files(): nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] - except (IndexError, AttributeError): - logging.debug('A node with no valid properties encountered and ignored') - -label_values = list(nodes_dict.values()) -duplicates = {label for label in label_values if label_values.count(label) > 1} -if duplicates: - logging.error(f"Duplicate node labels found: {sorted(duplicates)}") - quit() - -for edge in edges_text: + except (IndexError, AttributeError): + logging.debug('A node with no valid properties encountered and ignored') + +label_values = list(nodes_dict.values()) +duplicates = {label for label in label_values if label_values.count(label) > 1} +if duplicates: + logging.error(f"Duplicate node labels found: {sorted(duplicates)}") + quit() + +for edge in edges_text: try: data = edge.find('data', recursive=False) if data: @@ -785,13 +785,13 @@ def cleanup_script_files(): fcopy.write('CMD ["./a.out"]\n') # 7/02/21 fbuild.write('#!/bin/bash' + "\n") - for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.rsplit(".", 1) - dockerbuilddir = "docker-"+dockername.replace("/", "__").replace("\\", "__") - fbuild.write("mkdir "+dockerbuilddir+"\n") - fbuild.write("cd "+dockerbuilddir+"\n") + for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 + dockername,langext = sourcecode.rsplit(".", 1) + dockerbuilddir = "docker-"+dockername.replace("/", "__").replace("\\", "__") + fbuild.write("mkdir "+dockerbuilddir+"\n") + fbuild.write("cd "+dockerbuilddir+"\n") fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") #copy sourcefiles from ./src into corresponding directories fbuild.write("cp ../src/"+sourcecode+" .\n")