From c900ed3b25d358b49d7eaf6b6b10e1294a7e213d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 13 Nov 2023 23:20:16 +0100 Subject: [PATCH 1/9] add tests for #418 --- tests-ng/update-ignore.sh | 51 ++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/tests-ng/update-ignore.sh b/tests-ng/update-ignore.sh index 5128afe5..bed41a70 100755 --- a/tests-ng/update-ignore.sh +++ b/tests-ng/update-ignore.sh @@ -40,28 +40,45 @@ tmps=$(mktemp -d --suffix='-dotdrop-tests' || mktemp -d) tmpd=$(mktemp -d --suffix='-dotdrop-tests' || mktemp -d) dt="${tmps}/dotfiles" mkdir -p "${dt}" +clear_on_exit "${tmps}" +clear_on_exit "${tmpd}" -# dotfiles in dotdrop +# "a" dotfiles in dotdrop mkdir -p "${dt}"/a/{b,c,x} echo 'a' > "${dt}"/a/b/abfile echo 'a' > "${dt}"/a/c/acfile echo 'a' > "${dt}"/a/x/xfile +# "dir" dotfiles in dotdrop +mkdir -p "${dt}"/dir/{a,b,c} +echo 'a' > "${dt}"/dir/a/a +echo 'b' > "${dt}"/dir/b/b +echo 'c' > "${dt}"/dir/c/c + +# create destinations cp -r "${dt}"/a "${tmpd}"/ +cp -r "${dt}"/dir "${tmpd}"/ +# update "a" dotdrop files mkdir -p "${dt}"/a/be-gone echo 'a' > "${dt}"/a/be-gone/file -# filesystem files +# update "a" filesystem files touch "${tmpd}"/a/newfile echo 'b' > "${tmpd}"/a/c/acfile mkdir -p "${tmpd}"/a/newdir/b touch "${tmpd}"/a/newdir/b/c mkdir -p "${tmpd}"/a/x echo "b" > "${tmpd}"/a/x/xfile +echo "c" > "${tmpd}"/a/x/yfile -clear_on_exit "${tmps}" -clear_on_exit "${tmpd}" +# update "dir" filesystem +echo "new" > "${tmpd}"/dir/a/a +mkdir -p "${dt}"/dir/a/be-gone +touch "${tmpd}"/dir/newfile +mkdir -p "${tmpd}"/dir/ignore +echo "ignore-me" > "${tmpd}"/dir/ignore/ignore-me +echo 'ignore-me' > "${tmpd}"/dir/ignore-file # create the config file cfg="${tmps}/config.yaml" @@ -78,22 +95,38 @@ dotfiles: - "*/cfile" - "*/newfile" - "*/newdir" - - "*/x/**" + - "*/x/*" + d_dir: + dst: ${tmpd}/dir + src: dir + upignore: + - "*/ignore/*" + - "*/ignore-file" profiles: p1: dotfiles: - f_abc + - d_dir _EOF # update echo "[+] update" -cd "${ddpath}" | ${bin} update -f -c "${cfg}" --verbose --profile=p1 --key f_abc +cd "${ddpath}" | ${bin} update -f -c "${cfg}" --profile=p1 -# check files haven't been updated +# check "a" files are correct grep_or_fail 'b' "${dt}/a/c/acfile" grep_or_fail 'a' "${dt}/a/x/xfile" -[ -e "${dt}"/a/newfile ] && echo "should not have been updated" && exit 1 -[ -d "${dt}"/a/be-gone ] && echo "should have been removed" && exit 1 +[ -e "${dt}"/a/newfile ] && echo "'a' newfile should have been removed" && exit 1 +[ -d "${dt}"/a/be-gone ] && echo "'a' be-gone should have been removed" && exit 1 +[ -e "${dt}"/x/yfile ] && echo "'a' yfile should not have been added" && exit 1 + +# check "dir" files are correct +grep_or_fail 'new' "${dt}"/dir/a/a +[ -d "${dt}"/dir/a/be-gone ] && echo "'dir' be-gone should have been removed" && exit 1 +[ ! -e "${tmpd}"/dir/newfile ] && echo "'dir' newfile should have been removed" && exit 1 +[ -d "${dt}"/dir/ignore ] && echo "'dir' ignore dir not ignored" && exit 1 +[ -f "${dt}"/dir/ignore/ignore-me ] && echo "'dir' ignore-me not ignored" && exit 1 +[ -f "${dt}"/dir/ignore-file ] && echo "'dir' ignore-file not ignored" && exit 1 echo "OK" exit 0 From beef6a696f9f87ba6ae7507578b5dad6545ec79e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 13 Nov 2023 23:24:26 +0100 Subject: [PATCH 2/9] verbosity --- tests-ng/update-ignore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-ng/update-ignore.sh b/tests-ng/update-ignore.sh index bed41a70..ea1e28ef 100755 --- a/tests-ng/update-ignore.sh +++ b/tests-ng/update-ignore.sh @@ -111,7 +111,7 @@ _EOF # update echo "[+] update" -cd "${ddpath}" | ${bin} update -f -c "${cfg}" --profile=p1 +cd "${ddpath}" | ${bin} update -f --verbose -c "${cfg}" --profile=p1 # check "a" files are correct grep_or_fail 'b' "${dt}/a/c/acfile" From da25bfd0b9bdd82a71641619038612d985b43d61 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 13 Nov 2023 23:24:39 +0100 Subject: [PATCH 3/9] fix directory listed as copied even when ignored --- dotdrop/updater.py | 10 ++++++---- dotdrop/utils.py | 25 ++++++++++++++++++------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/dotdrop/updater.py b/dotdrop/updater.py index b8761b5e..149d352f 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -287,16 +287,18 @@ def _merge_dirs_create_left_only(self, diff, left, right, self.log.dry(f'would cp -r {exist} {new}') continue self.log.dbg(f'cp -r {exist} {new}') + cpied_cnt = 0 try: ign_func = self._ignore(ignores) - copytree_with_ign(exist, new, - ignore_func=ign_func, - debug=self.debug) + cpied_cnt= copytree_with_ign(exist, new, + ignore_func=ign_func, + debug=self.debug) except OSError as exc: msg = f'error copying dir {exist}' self.log.err(f'{msg}: {exc}') continue - self.log.sub(f'\"{new}\" dir added') + if cpied_cnt: + self.log.sub(f'\"{new}\" dir added') def _ignore(self, ignores): def ignore_func(path): diff --git a/dotdrop/utils.py b/dotdrop/utils.py index cd8baf2c..8d3ed67e 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -303,15 +303,18 @@ def must_ignore(paths, ignores, debug=False): def _cp(src, dst, ignore_func=None, debug=False): - """the copy function for copytree""" + """ + the copy function for copytree + returns the numb of files copied + """ if ignore_func and ignore_func(src): - return + return 0 if not os.path.isfile(src): # ignore special files if debug: LOG.dbg(f'ignore special file \"{src}\"', force=True) - return + return 0 dstdir = os.path.dirname(dst) if debug: LOG.dbg(f'mkdir \"{dstdir}\"', @@ -320,7 +323,10 @@ def _cp(src, dst, ignore_func=None, debug=False): if debug: LOG.dbg(f'cp {src} {dst}', force=True) - shutil.copy2(src, dst) + path = shutil.copy2(src, dst) + if os.path.exists(path): + return 1 + return 0 def copyfile(src, dst, debug=False): @@ -332,9 +338,13 @@ def copyfile(src, dst, debug=False): def copytree_with_ign(src, dst, ignore_func=None, debug=False): - """copytree with support for ignore""" + """ + copytree with support for ignore + returns the numb of files installed + """ if debug: LOG.dbg(f'copytree \"{src}\" to \"{dst}\"', force=True) + copied_count = 0 for entry in os.listdir(src): srcf = os.path.join(src, entry) dstf = os.path.join(dst, entry) @@ -346,12 +356,13 @@ def copytree_with_ign(src, dst, ignore_func=None, debug=False): LOG.dbg(f'mkdir \"{dstf}\"', force=True) os.makedirs(dstf, exist_ok=True) - copytree_with_ign(srcf, dstf, ignore_func=ignore_func) + copied_count += copytree_with_ign(srcf, dstf, ignore_func=ignore_func) else: if debug: LOG.dbg(f'copytree, copy file \"{src}\" to \"{dst}\"', force=True) - _cp(srcf, dstf, ignore_func=ignore_func, debug=debug) + copied_count += _cp(srcf, dstf, ignore_func=ignore_func, debug=debug) + return copied_count def uniq_list(a_list): From 2712eb536840d20c2e6e3cc333898bea3147e109 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 13 Nov 2023 23:28:13 +0100 Subject: [PATCH 4/9] linting --- dotdrop/updater.py | 6 +++--- dotdrop/utils.py | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dotdrop/updater.py b/dotdrop/updater.py index 149d352f..41197683 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -290,9 +290,9 @@ def _merge_dirs_create_left_only(self, diff, left, right, cpied_cnt = 0 try: ign_func = self._ignore(ignores) - cpied_cnt= copytree_with_ign(exist, new, - ignore_func=ign_func, - debug=self.debug) + cpied_cnt = copytree_with_ign(exist, new, + ignore_func=ign_func, + debug=self.debug) except OSError as exc: msg = f'error copying dir {exist}' self.log.err(f'{msg}: {exc}') diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 8d3ed67e..2704dbbe 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -356,12 +356,17 @@ def copytree_with_ign(src, dst, ignore_func=None, debug=False): LOG.dbg(f'mkdir \"{dstf}\"', force=True) os.makedirs(dstf, exist_ok=True) - copied_count += copytree_with_ign(srcf, dstf, ignore_func=ignore_func) + copied_count += copytree_with_ign(srcf, + dstf, + ignore_func=ignore_func) else: if debug: LOG.dbg(f'copytree, copy file \"{src}\" to \"{dst}\"', force=True) - copied_count += _cp(srcf, dstf, ignore_func=ignore_func, debug=debug) + copied_count += _cp(srcf, + dstf, + ignore_func=ignore_func, + debug=debug) return copied_count From 6d2c5ea59f177b1a2b14835c46b4c63048308b85 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 14 Nov 2023 21:26:15 +0100 Subject: [PATCH 5/9] check for copy return value --- dotdrop/importer.py | 12 ++++++++---- dotdrop/installer.py | 14 ++++++++++---- dotdrop/utils.py | 3 ++- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/dotdrop/importer.py b/dotdrop/importer.py index 7ecf61c8..cd9388f5 100644 --- a/dotdrop/importer.py +++ b/dotdrop/importer.py @@ -249,15 +249,19 @@ def _import_to_dotpath(self, in_dotpath, in_fs, trans_update=None): if not os.path.isdir(in_fs): # is a file self.log.dbg(f'{in_fs} is file') - copyfile(in_fs, srcf, debug=self.debug) + if not copyfile(in_fs, srcf, debug=self.debug): + self.log.err(f'importing \"{in_fs}\" failed') + return False else: # is a dir if os.path.exists(srcf): shutil.rmtree(srcf) self.log.dbg(f'{in_fs} is dir') - copytree_with_ign(in_fs, srcf, - ignore_func=self._ignore, - debug=self.debug) + if not copytree_with_ign(in_fs, srcf, + ignore_func=self._ignore, + debug=self.debug): + self.log.err(f'importing \"{in_fs}\" failed') + return False except shutil.Error as exc: in_dotpath = exc.args[0][0][0] why = exc.args[0][0][2] diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 39a2b181..22666a6f 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -484,7 +484,8 @@ def _symlink(self, src, dst, actionexec=None, absolute=True): # remove symlink if self.backup and not os.path.isdir(dst): - self._backup(dst) + if not self._backup(dst): + return False, f'could not backup {dst}' overwrite = True try: removepath(dst) @@ -770,7 +771,8 @@ def _write(self, src, dst, content=None, overwrite = True if self.backup: - self._backup(dst) + if not self._backup(dst): + return False, f'could not backup {dst}' else: self.log.dbg(f'file does not exist on filesystem: {dst}') @@ -869,16 +871,20 @@ def _create_dirs(self, directory): def _backup(self, path): """backup file pointed by path""" if self.dry: - return + return True dst = path.rstrip(os.sep) + self.backup_suffix self.log.log(f'backup {path} to {dst}') # os.rename(path, dst) # copy to preserve mode on chmod=preserve # since we expect dotfiles this shouldn't have # such a big impact but who knows. - copyfile(path, dst, debug=self.debug) + if not copyfile(path, dst, debug=self.debug): + return False + if not os.path.exists(dst): + return False stat = os.stat(path) os.chown(dst, stat.st_uid, stat.st_gid) + return True def _exec_pre_actions(self, actionexec): """execute action executor""" diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 2704dbbe..968eb4ab 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -333,8 +333,9 @@ def copyfile(src, dst, debug=False): """ copy file from src to dst no dir expected! + returns True if file was copied """ - _cp(src, dst, debug=debug) + return _cp(src, dst, debug=debug) == 1 def copytree_with_ign(src, dst, ignore_func=None, debug=False): From 18e326b14bba0abe88dc9c61b0a72b31409b1928 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 14 Nov 2023 21:26:25 +0100 Subject: [PATCH 6/9] only check links on ci/cd --- scripts/check-doc.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/check-doc.sh b/scripts/check-doc.sh index ddd1a8e9..bb72a9f1 100755 --- a/scripts/check-doc.sh +++ b/scripts/check-doc.sh @@ -21,10 +21,15 @@ if ! which remark >/dev/null 2>&1; then exit 1 fi -echo "------------------------" -echo "checking internal links" -find . -type f -iname '*.md' | while read -r line; do - remark -f -u validate-links "${line}" -done +in_cicd="${GH_WORKFLOW:-}" +if [ -n "${in_cicd}" ]; then + echo "------------------------" + echo "checking internal links" + find . -type f -iname '*.md' | while read -r line; do + remark -f -u validate-links "${line}" + done +else + echo "not checking internal links..." +fi echo "documentation OK" From 61059b6740bb7aee4c012ed6eedd077371796747 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 16 Nov 2023 07:11:41 +0100 Subject: [PATCH 7/9] coveralls --- scripts/check_links.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/check_links.py b/scripts/check_links.py index 4ba0ec3f..34d9d454 100755 --- a/scripts/check_links.py +++ b/scripts/check_links.py @@ -31,6 +31,7 @@ ] IGNORES = [ 'badgen.net', + 'coveralls.io', ] OK_WHEN_FORBIDDEN = [ 'linux.die.net' From ee697148241d18eebad926ce189e5ffd8ce8663b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 16 Nov 2023 17:06:15 +0100 Subject: [PATCH 8/9] coveralls token --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f73de8c8..f79a3014 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -39,3 +39,4 @@ jobs: coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} From e0d0321c1fb738183f14b0474e1476f9b5416061 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 16 Nov 2023 21:50:58 +0100 Subject: [PATCH 9/9] coveralls version --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f79a3014..24044107 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -36,6 +36,7 @@ jobs: - name: Coveralls run: | pip install coveralls + coveralls --version coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}