Linux utils

Split huge file in place

Taken from: https://superuser.com/questions/177823/are-there-any-tools-in-linux-for-splitting-a-file-in-place

Save following as ./splitmb.sh:

#!/bin/bash
# (c) whitequark 2010
# (c) dertalai 2015 (minimal modifications)

set -e

if [ $# != 2 ]; then
  echo "Usage: $0  "
  echo "  This script will split file to multiple parts, starting from"
  echo "  the end, and truncating the original file in process."
  echo "  Part size is specified in megabytes (1 MB = 1048576 bytes)."
  echo "  Use at your own risk."
  exit 0
fi

filename=$1
#partsize=$2
partsizeMB=$2
partsize=$(($2 * 1048576))

size=$(stat -c '%s' "${filename}")
parts=$(($size / $partsize))

do_split() {
  _part=$1
  _size=$2

  echo "Splitting part $_part"
  echo $(($partsize * ($_part - 1)))
  dd if="${filename}" of="${filename}.$(printf '%04d' $_part)" \
      count=$partsizeMB bs=1M skip=$((($_part - 1) * $partsizeMB))
  echo "Truncating source file"
  truncate "${filename}" --size="-$_size"
}

lastsize=$(($size % $partsize))
if [ $lastsize != 0 ]; then
  do_split $(($parts + 1)) $lastsize
fi

for i in $(seq $parts -1 1); do
  do_split $i $partsize
done

rm "${filename}"

make executable

Split into 100GB chunks: ./splitmb.sh ./huge.file.bin 100000

Check directory size, alarm after reaching size

while true; do size=`du -sh -B1G ./directory | cut -f 1 | cut -d '.' -f 1 | cut -d 'G' -f 1`; echo -n "$size "; if [[ $size -gt <gigs> ]]; then printf 'yes\n\a'; else printf 'no\n'; fi; sleep 60; done