dvc.yaml

vars:
  # where to clone Git repositories (should be external directory)
  - repos_dir: ~/example_repositories
  # list of Git repositories to clone, and their URLs
  - repos:
      - name: tensorflow
        url: https://github.com/tensorflow/tensorflow.git
      - name: qtile
        url: https://github.com/qtile/qtile.git
      - name: linux
        url: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
      - name: git
        # Git Source Code Mirror
        url: https://github.com/git/git.git
  # repositories, and authors in those repositories, to analyze/annotate
  - authors:
      - repo: tensorflow
        nick: ezhulenev
        annotate_args: ''
        args: --author=ezhulenev@google.com
      - repo: tensorflow
        nick: yong.tang
        annotate_args: ''
        args: --author=yong.tang.github@outlook.com
      - repo: qtile
        nick: all_authors-no_merges
        annotate_args: >-
          --ext-to-language=".hgignore:Ignore List"
          --filename-to-language=".hgignore:Ignore List"
          --filename-to-language=README:Text
          --filename-to-language=PACKAGING:Text
          --filename-to-language=BUILDING:Text
          --filename-to-language=changelog:Text
        # --max-parents=1 is the same as --no-merges
        args: --max-parents=1
      - repo: linux
        nick: all_authors-last_release
        # using folded style (denoted by “>”), with strip chomping mode (denoted by "-")
        # we could have maybe used DVC dictionary unpacking instead
        # https://dvc.org/doc/user-guide/project-structure/dvcyaml-files#dictionary-unpacking
        annotate_args: >-
          --ext-to-language=.rs:Rust
          --filename-to-language=Makefile.lib:Makefile
          --filename-to-language=Makefile.config:Makefile
          --filename-to-language=Makefile.deps:Makefile
          --filename-to-language=Makefile.perf:Makefile
          --filename-to-language=Makefile.ubsan:Makefile
          --pattern-to-purpose=Makefile.*:project
          --filename-to-language=Kconfig:Kconfig
          --filename-to-language=Kconfig.debug:Kconfig
          --pattern-to-purpose=Kconfig*:project
          --ext-to-language=".S:Unix Assembly"
          --ext-to-language=".rules:Udev Rules File"
          --filename-to-language=README:Text
          --filename-to-language=CONTRIBUTORS:Text
          --filename-to-language=MAINTAINERS:Text
          --ext-to-language=".dts:Device Tree Source"
          --ext-to-language=".dtsi:Device Tree Source include"
          --ext-to-language=".dtso:Device Tree Source overlay"
          --pattern-to-purpose=*.dts:data
          --pattern-to-purpose=*.dts?:data
        args:  --max-parents=1 v6.8..v6.9
      - repo: git
        nick: jnareb
        annotate_args: >-
          --ext-to-language=.cgi:Perl
          --filename-to-language=config.mak.in:Makefile
          --filename-to-language="git.spec.in:RPM Spec"
        args: --max-parents=1 --author=jnareb
      - repo: linux
        nick: all_authors-2021_2023
        annotate_args: >-
          --ext-to-language=.rs:Rust
          --filename-to-language=Makefile.lib:Makefile
          --filename-to-language=Makefile.config:Makefile
          --filename-to-language=Makefile.deps:Makefile
          --filename-to-language=Makefile.perf:Makefile
          --filename-to-language=Makefile.ubsan:Makefile
          --pattern-to-purpose=Makefile.*:project
          --filename-to-language=Kconfig:Kconfig
          --filename-to-language=Kconfig.debug:Kconfig
          --pattern-to-purpose=Kconfig*:project
          --ext-to-language=".S:Unix Assembly"
          --ext-to-language=".rules:Udev Rules File"
          --filename-to-language=README:Text
          --filename-to-language=CONTRIBUTORS:Text
          --filename-to-language=MAINTAINERS:Text
          --ext-to-language=".dts:Device Tree Source"
          --ext-to-language=".dtsi:Device Tree Source include"
          --ext-to-language=".dtso:Device Tree Source overlay"
          --pattern-to-purpose=*.dts:data
          --pattern-to-purpose=*.dts?:data
        args: --max-parents=1 --after=2021.01.01 --before=2023.12.31
      - repo: linux
        nick: all_authors-2021_2023-no_repo
        annotate_args: >-
          --ext-to-language=.rs:Rust
          --filename-to-language=Makefile.lib:Makefile
          --filename-to-language=Makefile.config:Makefile
          --filename-to-language=Makefile.deps:Makefile
          --filename-to-language=Makefile.perf:Makefile
          --filename-to-language=Makefile.ubsan:Makefile
          --pattern-to-purpose=Makefile.*:project
          --filename-to-language=Kconfig:Kconfig
          --filename-to-language=Kconfig.debug:Kconfig
          --pattern-to-purpose=Kconfig*:project
          --ext-to-language=".S:Unix Assembly"
          --ext-to-language=".rules:Udev Rules File"
          --filename-to-language=README:Text
          --filename-to-language=CONTRIBUTORS:Text
          --filename-to-language=MAINTAINERS:Text
          --ext-to-language=".dts:Device Tree Source"
          --ext-to-language=".dtsi:Device Tree Source include"
          --ext-to-language=".dtso:Device Tree Source overlay"
          --pattern-to-purpose=*.dts:data
          --pattern-to-purpose=*.dts?:data
        args: --no-use-repo --max-parents=1 --after=2021.01.01 --before=2023.12.31

stages:
  clone:
    foreach: ${repos}
    do:
      desc: 'Clone ${item.name} repository from ${item.url}'
      cmd:
        - >-
          if [ -d ${repos_dir}/${item.name} ]; then
             echo 'Repository ${item.name} already cloned, skipping';
          else
             git clone ${item.url} ${repos_dir}/${item.name};
          fi
        - echo ${item.url} >data/examples/${item.name}.url
      outs:
        - data/examples/${item.name}.url

  annotate:
    foreach: ${authors}
    do:
      desc: 'Generating annotations for ${item.nick} from-repo ${item.repo}'
      cmd: >-
        diff-annotate ${item.annotate_args} from-repo
        --output-dir=data/examples/annotations/${item.repo}/${item.nick}/
        ${repos_dir}/${item.repo}
        ${item.args}
      deps:
        - src/diffannotator/annotate.py
        - data/examples/${item.repo}.url
      outs:
        - data/examples/annotations/${item.repo}/${item.nick}/

  purpose-counter:
    foreach: ${repos}
    do:
      desc: 'Running `diff-gather-stats purpose-counter ...` on ${item.name} repo'
      cmd: >-
        diff-gather-stats --annotations-dir=''
        purpose-counter
        --output=data/examples/stats/${item.name}.purpose-counter.json
        data/examples/annotations/${item.name}/
      deps:
        - data/examples/annotations/${item.name}
        - src/diffannotator/gather_data.py
      outs:
        - data/examples/stats/${item.name}.purpose-counter.json

  purpose-per-file:
    foreach: ${repos}
    do:
      desc: 'Running `diff-gather-stats purpose-per-file ...` on ${item.name} repo'
      cmd: >-
        diff-gather-stats --annotations-dir=''
        purpose-per-file
        data/examples/stats/${item.name}.purpose-per-file.json
        data/examples/annotations/${item.name}/
      deps:
        - data/examples/annotations/${item.name}
        - src/diffannotator/gather_data.py
      outs:
        - data/examples/stats/${item.name}.purpose-per-file.json

  lines-stats:
    foreach: ${repos}
    do:
      desc: 'Running `diff-gather-stats lines-stats ...` on ${item.name} repo'
      cmd: >-
        diff-gather-stats --annotations-dir=''
        lines-stats
        data/examples/stats/${item.name}.lines-stats.json
        data/examples/annotations/${item.name}/
      deps:
        - data/examples/annotations/${item.name}
        - src/diffannotator/gather_data.py
      outs:
        - data/examples/stats/${item.name}.lines-stats.json

  lines-stats.purpose-to-type:
    foreach: ${repos}
    do:
      desc: 'Running `diff-gather-stats lines-stats --purpose-to-annotation...` on ${item.name} repo'
      cmd: >-
        diff-gather-stats --annotations-dir=''
        lines-stats
        --purpose-to-annotation=data
        --purpose-to-annotation=documentation
        --purpose-to-annotation=markup
        --purpose-to-annotation=other
        --purpose-to-annotation=project
        --purpose-to-annotation=test
        data/examples/stats/${item.name}.lines-stats.purpose-to-type.json
        data/examples/annotations/${item.name}/
      deps:
        - data/examples/annotations/${item.name}
        - src/diffannotator/gather_data.py
      outs:
        - data/examples/stats/${item.name}.lines-stats.purpose-to-type.json

  timeline:
    foreach: ${repos}
    do:
      desc: 'Running `diff-gather-stats timeline ...` on ${item.name} repo'
      cmd: >-
        diff-gather-stats --annotations-dir=''
        timeline
        data/examples/stats/${item.name}.timeline.json
        data/examples/annotations/${item.name}/
      deps:
        - data/examples/annotations/${item.name}
        - src/diffannotator/gather_data.py
      outs:
        - data/examples/stats/${item.name}.timeline.json

  timeline.purpose-to-type:
    foreach: ${repos}
    do:
      desc: 'Running `diff-gather-stats timeline --purpose-to-annotation...` on ${item.name} repo'
      cmd: >-
        diff-gather-stats --annotations-dir=''
        timeline
        --purpose-to-annotation=data
        --purpose-to-annotation=documentation
        --purpose-to-annotation=markup
        --purpose-to-annotation=other
        --purpose-to-annotation=project
        --purpose-to-annotation=test
        data/examples/stats/${item.name}.timeline.purpose-to-type.json
        data/examples/annotations/${item.name}/
      deps:
        - data/examples/annotations/${item.name}
        - src/diffannotator/gather_data.py
      outs:
        - data/examples/stats/${item.name}.timeline.purpose-to-type.json