Move CI workflow to common script, simplify packaging script, remove …

…packing into a tarball since CI already packages artifacts into an archive

Internal-tag: [#57771]
Signed-off-by: Krzysztof Obłonczek <[email protected]>

.ci.yml

@@ -33,6 +33,16 @@ examples:
     paths:
       - examples/**/build
 
+package_cores:
+  stage: test
+  tags: ['ace-x86_64']
+  image: debian:bookworm
+  script:
+    - ./.github/scripts/ci.sh package_cores
+  artifacts:
+    paths:
+      - core_repo/**
+
 build_docs:
   image: $CI_DOCS_DOCKER_IMAGE
   stage: build_docs

.flake8

@@ -9,7 +9,6 @@ exclude =
     .eggs,
     .git,
     .gitignore,
-    .github,
     .nox,
     .pytest_cache,
     __pycache__,

.github/scripts/ci.sh

@@ -134,6 +134,31 @@ generate_examples() {
     done
 }
 
+package_cores() {
+    install_common_system_packages
+    begin_command_group "Install system packages for packaging cores"
+    log_cmd apt-get install -y --no-install-recommends python3-dev
+    end_command_group
+    enter_venv
+
+    begin_command_group "Install python packages for packaging cores"
+    log_cmd pip install git+https://github.com/antmicro/tuttest
+    end_command_group
+
+    install_topwrap
+
+    begin_command_group "Install Topwrap's parsing dependencies"
+    log_cmd pip install ".[topwrap-parse]"
+    end_command_group
+
+    begin_command_group "Package cores for release"
+    log_cmd mkdir core_repo
+    log_cmd pushd core_repo
+    log_cmd python ../.github/scripts/package_cores.py
+    log_cmd popd
+    end_command_group
+}
+
 case "$1" in
 lint)
     run_lint
@@ -144,7 +169,10 @@ tests)
 examples)
     generate_examples
     ;;
+package_cores)
+    package_cores
+    ;;
 *)
-    echo "Usage: $0 {lint|tests|examples}"
+    echo "Usage: $0 {lint|tests|examples|package_cores}"
     ;;
 esac

.github/scripts/package_cores.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2024 Antmicro <www.antmicro.com>
+# SPDX-License-Identifier: Apache-2.0
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List
+
+from topwrap.repo.files import HttpGetFile
+from topwrap.repo.user_repo import UserRepo, VerilogFileHandler
+
+
+@dataclass
+class RemoteRepo:
+    name: str
+    root_url: str
+    sources: List[str]
+
+
+repos = [
+    RemoteRepo(
+        name="vexriscv",
+        root_url="https://raw.githubusercontent.com/litex-hub/pythondata-cpu-vexriscv/1979a644dbe64d8d32dfbdd970dccee6add63723/pythondata_cpu_vexriscv/verilog",
+        sources=[
+            "VexRiscv.v",
+        ],
+    ),
+]
+
+
+def package_cores():
+    """Generates reusable cores package for usage in Topwrap project."""
+    core_repo = UserRepo()
+
+    for repo in repos:
+        core_files = []
+        for file in repo.sources:
+            core_files.append(HttpGetFile(f"{repo.root_url}/{file}"))
+
+        core_repo.add_files(VerilogFileHandler(core_files))
+
+        Path(repo.name).mkdir(exist_ok=True)
+        core_repo.save(repo.name)
+
+
+if __name__ == "__main__":
+    package_cores()

.github/workflows/pipeline.yml

@@ -53,7 +53,7 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: debian:bookworm
-    name: 'Generate examples'
+    name: "Generate examples"
 
     steps:
       - name: Install git package
@@ -72,3 +72,27 @@ jobs:
         with:
           name: examples
           path: examples/**/build
+
+  package_cores:
+    runs-on: ubuntu-latest
+    container:
+      image: debian:bookworm
+    name: "Package cores repository"
+
+    steps:
+      - name: Install git package
+        run: |
+          apt-get update -qq
+          apt-get install -y git
+
+      - uses: actions/checkout@v4
+
+      - name: Pack cores into a Topwrap repository
+        run: |
+          ./.github/scripts/ci.sh package_cores
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: core_repo
+          path: core_repo/**

.gitignore

@@ -25,7 +25,6 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
-*.tar.gz
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

docs/source/cli.md

@@ -72,13 +72,3 @@ To get help, use:
 ```
 python -m topwrap [build|kpm_client|parse] --help
 ```
-
-## Generating IP core archive
-
-HDL sources can be packaged into a reusable tarball with autogenerated IP core description YAMLs:
-
-```
-python -m topwrap pack_core NAME SOURCE_DIRS
-```
-
-Where `NAME` is the name of target archive and `SOURCE_DIRS` is one or more directories with HDL files to be included in the package.

pyproject.toml

@@ -75,7 +75,6 @@ exclude = '''
     | \.eggs
     | \.git
     | \.gitignore
-    | \.github
     | \.nox
     | \.pytest_cache
     | __pycache__
@@ -94,7 +93,6 @@ skip = [
     ".eggs",
     ".git",
     ".gitignore",
-    ".github",
     ".nox",
     ".pytest_cache",
     "__pycache__",