Merge pull request #37 from k2kobayashi/results

- modify result calculation

crank/bin/evaluate_mcd.py

@@ -82,28 +82,16 @@ def calculate(cv_path, gt_file_list, conf, spkr_conf):
 def main():
 
     parser = argparse.ArgumentParser(description="calculate MCD.")
-    parser.add_argument("--conf", type=str, required=True, help="Configuration file")
+    parser.add_argument("--conf", type=str, help="configuration file")
+    parser.add_argument("--spkr_conf", type=str, help="speaker configuration file")
     parser.add_argument(
-        "--spkr_conf", type=str, required=True, help="Speaker configuration file"
+        "--featdir", type=str, help="root directory of ground truth h5",
     )
+    parser.add_argument("--outwavdir", type=str, help="converted waveform directory")
     parser.add_argument(
-        "--featdir",
-        type=str,
-        required=True,
-        help="Root directory of ground truth feature h5 files",
-    )
-    parser.add_argument(
-        "--outwavdir", type=str, required=True, help="Converted waveform directory"
-    )
-    parser.add_argument(
-        "--out",
-        "-O",
-        type=str,
-        help="The output filename. " "If omitted, then output to sys.stdout",
-    )
-    parser.add_argument(
-        "--n_jobs", default=1, type=int, help="number of parallel jobs"
+        "--out", type=str, help="if omitted, then output to sys.stdout",
     )
+    parser.add_argument("--n_jobs", default=1, type=int, help="number of parallel jobs")
     args = parser.parse_args()
 
     # logging info
@@ -121,7 +109,7 @@ def main():
     if conf["output_feat_type"] == "mcep":
         converted_files = sorted(list(Path(args.outwavdir).glob("*.h5")))
     else:
-        converted_files = sorted(list(Path(args.outwavdir).glob("*.wav")))
+        converted_files = sorted(list(Path(args.outwavdir).rglob("*.wav")))
     logging.info(f"number of utterances = {len(converted_files)}")
 
     # load ground truth scp
@@ -144,7 +132,7 @@ def main():
     pairwise_MCD = {}
     for k, v in MCD_list:
         orgspk, tarspk, _ = k.split("-")
-        pair = orgspk + "-" + tarspk
+        pair = orgspk + " " + tarspk
         if pair not in pairwise_MCD:
             pairwise_MCD[pair] = []
         pairwise_MCD[pair].append(v)

crank/bin/evaluate_mosnet.py

@@ -20,14 +20,9 @@ def main():
         description="Use MOSnet to predict quality scores.",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
+    parser.add_argument("--outwavdir", type=str, help="Converted waveform directory")
     parser.add_argument(
-        "--outwavdir", type=str, required=True, help="Converted waveform directory"
-    )
-    parser.add_argument(
-        "--out",
-        "-O",
-        type=str,
-        help="The output filename. " "If omitted, then output to sys.stdout",
+        "--out", type=str, help="If omitted, then output to sys.stdout",
     )
     args = parser.parse_args()
 
@@ -39,7 +34,7 @@ def main():
     )
 
     # load converted files.
-    converted_files = sorted(list(Path(args.outwavdir).glob("*.wav")))
+    converted_files = sorted(list(Path(args.outwavdir).rglob("*.wav")))
     logging.info(f"number of utterances = {len(converted_files)}")
 
     # construct metric class
@@ -65,7 +60,7 @@ def main():
     pairwise_scores = {}
     for k, v in scores.items():
         orgspk, tarspk, _ = k.split("-")
-        pair = orgspk + "-" + tarspk
+        pair = orgspk + " " + tarspk
         if pair not in pairwise_scores:
             pairwise_scores[pair] = []
         pairwise_scores[pair].append(v)

crank/bin/rename_decoded.py

@@ -0,0 +1,33 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+#
+# Copyright (c) 2021 Kazuhiro KOBAYASHI <[email protected]>
+#
+# Distributed under terms of the MIT license.
+
+"""
+
+"""
+
+import argparse
+from pathlib import Path
+
+
+def main():
+    # options for python
+    description = "Rename decoded waveforms"
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument("--outwavdir", type=str, help="decoded waveform directory")
+    args = parser.parse_args()
+    decoded_files = Path(args.outwavdir).glob("*.wav")
+
+    for f in decoded_files:
+        stem = str(f.stem).rstrip("_gen")
+        org = stem.split("org")[1].split("cv")[0].lstrip("-").rstrip("_")
+        (f.parent / org).mkdir(exist_ok=True, parents=True)
+        f.rename((f.parent / org / (stem + ".wav")))
+
+
+if __name__ == "__main__":
+    main()

crank/bin/train.py

@@ -125,7 +125,7 @@ def main():
     parser = argparse.ArgumentParser(description=description)
     parser.add_argument("--flag", help='flag ["train", "eval", "reconstruction"]')
     parser.add_argument("--n_jobs", type=int, default=-1, help="# of CPUs")
-    parser.add_argument("--conf", type=str, help="ymal file for network parameters")
+    parser.add_argument("--conf", type=str, help="yaml file for network parameters")
     parser.add_argument("--checkpoint", type=str, default=None, help="Resume")
     parser.add_argument("--scpdir", type=str, help="scp directory")
     parser.add_argument("--featdir", type=str, help="output feature directory")

egs/vaevc/template/path.sh

@@ -14,6 +14,7 @@ export CUDA_DEVICE_ORDER=PCI_BUS_ID
 export OMP_NUM_THREADS=1
 export PYTHONIOENCODING=UTF-8
 export MPL_BACKEND=Agg
+export TF_CPP_MIN_LOG_LEVEL=3
 
 # check installation
 if ! python -c "import crank" > /dev/null 2>&1 ; then

egs/vaevc/template/run.sh

@@ -227,7 +227,11 @@ if [ "${stage}" -le 6 ] && [ "${stop_stage}" -ge 6 ]; then
         echo "successfully finished decoding."
 
         # rename
-        find "${outdir}/wav" -name '*_gen.wav' | sed -e "p;s/_gen//" | xargs -n2 mv
+        echo "Rename decoded files "
+        ${train_cmd} "${outdir}/rename_decoded.log" \
+            python -m crank.bin.rename_decoded  \
+                --outwavdir "${outdir}"/wav
+        echo "successfully renamed decoded waveforms."
     else
         echo "Not supported decoder type. GL and PWG are available."
     fi

egs/vaevc/vcc2018v1/RESULTS.md

@@ -0,0 +1,12 @@
+# vcc2018v1: mlfb_vqvae.conf
+- mcd
+  - 9.54667
+- mosnet
+  - 3.42398
+---
+# vcc2018v1: mlfb_cycle.conf
+- mcd
+  - 9.44014
+- mosnet
+  - 3.46762
+---

tools/requirements.txt

@@ -12,3 +12,4 @@ torch-optimizer
 pytorch_lamb
 tensorflow-gpu==2.4.0
 gdown
+museval

utils/generate_results.sh

@@ -0,0 +1,39 @@
+#! /bin/bash
+#
+# generate_results.sh
+# Copyright (C) 2020 Kazuhiro KOBAYASHI <[email protected]>
+#
+# Distributed under terms of the MIT license.
+#
+
+# shellcheck disable=SC1091
+. ./path.sh || exit 1
+
+. utils/parse_options.sh || exit 1;
+
+conf=$1
+voc_expdir=$2
+# check arguments
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 <conf>"
+    exit 1
+fi
+
+set -eu
+
+# shellcheck disable=SC2012
+recipe=$(basename "$PWD")
+confname=$(basename "${conf}" .yml)
+voc_confname=$(basename "${voc_expdir}")
+PWG_dir=exp/${confname}/eval_${voc_confname}_wav
+# shellcheck disable=SC2012
+result_dir=$PWG_dir/$(basename "$(ls -dt "${PWG_dir}"/* | head -1)")
+
+mcd=$(head -n -2 < "$result_dir"/mcd.log | tac | head -n -4 | tac | awk '{if($1!=$2) print $1, $2, $3}' | awk '{sum+=$3} END {print sum/NR}')
+mosnet=$(head -n -2 "$result_dir"/mosnet.log | tac | head -n -8 | tac | awk '{if($1!=$2) print $1, $2, $3}' | awk '{sum+=$3} END {print sum/NR}')
+echo "# $recipe: $confname.conf"
+echo "- mcd"
+echo "  - $mcd"
+echo "- mosnet"
+echo "  - $mosnet"
+echo "---"