dedupe

#!/usr/bin/env sh
# compat: -ash +bash +dash +hush +ksh +mksh +oksh +osh +posh +yash +zsh

dedupe() ( ### @- copy a directory, but make hard/softlinks for identical files.
# Copyright (C) 2022 Connor Olding
h=0 usage='dedupe [OPTION...] {SOURCE} {TARGET} [FILE...]
Copy files, recursively, from the source directory to the target directory,
creating hard (or soft) links to any earlier file containing the same data.
Optional file arguments allow for finer control of the order of files seen.

      --help                 display this help and exit
      --soft                 create soft links (default)
      --hard                 create hard links

This software is provided WITHOUT WARRANTY under the terms of the ISC license.'

read program <<'EOF'
BEGIN{if(!D){exit 64}E=(Q="'")"\\"'\\""Q;H=(G=".")(F="/");I=" -- "Q;J="[^/]*/";\
sub(F"?$",F,D)gsub(Q,E,D)}function C(x){if(X=system(x))exit X}{if((p=$2)==B)\
{print V;exit 65}for(r=z=i=d=B;d!=!i;i++){b=a=p;d=sub("/.*",B,a);sub(J,B,p);\
if(a==G G)e=!sub(J,B,z);else z=a==G?z:b!=a?z a F:z a;if(e||!i&&!a){print W;exit\
 65}}P=p=!z?H:z;if((o=O[$1])==p)next;gsub(Q,E,P);M=m=D P;gsub("/[^/]+$",B,M);\
if(!u[M]++)C("mkdir -p"I M Q);if(o){T=split(o,t,F);S=split(p,s,F);for(i=1;t[i]\
==s[i];i++);for(j=i;j<S;j++)r=r G H;for(j=i;j<T;j++)r=r t[j]F;r=h?o:r t[T];gsub\
(Q,E,r);C("ln -f"(h?I D:" -s"I)r"' '"m Q);next}O[$1]=p;C("cp -p"I P"' '"m Q)}
EOF

die() { printf '\033[1mdedupe:\033[m %s\n' >&2 "$2"; exit "$1" ;}

while o="${1#-}" && [ "${#o}" != "${#1}" ] && shift
    do case "$o" in
    (-) break ;;
    (-help) printf '\033[1mUsage:\033[m %s\n' "$usage"; exit ;;
    (-soft) h=0 ;; (-hard) h=1 ;;
    (-*) die 64 "unknown long option: -$o" ;;
    (*) die 64 "unknown short option: -${o%"${o#?}"}" ;;
    esac; done &&
{ [ -d /tmp ] || die 72 "/tmp must be mounted" ;} &&
{ [ $# -ge 2 ] || die 64 'expected at least 2 arguments' ;} &&
unset CDPATH && D="$(readlink -f "$2")" && cd -- "$1" && shift 2 &&
T="$(mktemp -d)" && t="$T/sums" && { [ $# -eq 0 ] || sha256sum >"$t" "$@" ;} &&
find ! -type d ! -type s -exec sha256sum -- {} + >>"$t~" &&
sort -k2 <"$t~" >>"$t" && awk -v D="$D" -v h="$h" -F'  ' "$program" <"$t"; x=$?
! [ -d "$t" ] || rm -r "$t" && [ $x = 0 ] || die $x 'exiting after failure'
)

[ -n "${preload+-}" ] || dedupe "$@"