Python

This Python script renames files and directories in a given folder recursively, removing a given string from filenames and directory names. It does not require any libraries to be installed.

Note that this script has been tested, however we can’t guarantee it won’t destroy some data under some conditions. Backup your data and use with caution (best to try using --dry --verbose!)

These are the command line options it provides:

usage: RemoveFromNames.py [-h] [-c] [--dry] [-D] [-F] [-i] [-v] dir remove_string

Process and rename files and directories by removing a specific string.

positional arguments:
  dir                   Input directory to scan recursively
  remove_string         String to remove from filenames and folder names

options:
  -h, --help            show this help message and exit
  -c, --compress_spaces
                        Compress multiple spaces into one
  --dry                 Dry run. Do not actually rename anything.
  -D, --no_dirs         Don't rename directories.
  -F, --no_files        Don't rename files.
  -i, --case_insensitive
                        Perform case-insensitive compare and replace.
  -v, --verbose         Print what is being renamed.

Source code:

#!/usr/bin/env python3
import os
import argparse

def rename_files_in_directory(base_dir, remove_string, compress_spaces=False, dry_run=False, verbose=False, no_dirs=False, no_files=False, case_insensitive=False):
    if case_insensitive:
        def contains(sub, main):
            return sub.lower() in main.lower()
        
        def replace(source, target, string):
            pattern = re.compile(re.escape(source), re.IGNORECASE)
            return pattern.sub(target, string)
    else:
        contains = lambda sub, main: sub in main
        replace = lambda source, target, string: string.replace(source, target)

    # First, rename directories if not skipped by --no_dirs
    if not no_dirs:
        for dirpath, dirnames, _ in os.walk(base_dir, topdown=False):
            for dirname in dirnames:
                if contains(remove_string, dirname):
                    new_name = replace(remove_string, "", dirname)
                    if compress_spaces:
                        new_name = ' '.join(new_name.split())
                    new_name = new_name.strip()

                    if os.path.exists(os.path.join(dirpath, new_name)):
                        print(f"Cannot rename directory '{dirname}' to '{new_name}' in '{dirpath}' because a directory with the new name already exists.")
                        continue

                    if verbose:
                        print(f"Renaming directory '{dirname}' to '{new_name}' in '{dirpath}'")
                    
                    if not dry_run:
                        os.rename(os.path.join(dirpath, dirname), os.path.join(dirpath, new_name))

    # Then, rename files if not skipped by --no_files
    if not no_files:
        for dirpath, _, filenames in os.walk(base_dir):
            for filename in filenames:
                if contains(remove_string, filename):
                    new_name = replace(remove_string, "", filename)
                    if compress_spaces:
                        new_name = ' '.join(new_name.split())
                    new_name = new_name.strip()
                    
                    if os.path.exists(os.path.join(dirpath, new_name)):
                        print(f"Cannot rename '{filename}' to '{new_name}' in '{dirpath}' because a file with the new name already exists.")
                        continue
                    
                    if verbose:
                        print(f"Renaming file '{filename}' to '{new_name}' in '{dirpath}'")
                    
                    if not dry_run:
                        os.rename(os.path.join(dirpath, filename), os.path.join(dirpath, new_name))

def main():
    parser = argparse.ArgumentParser(description="Process and rename files and directories by removing a specific string.")
    
    parser.add_argument("dir", type=str, help="Input directory to scan recursively")
    parser.add_argument("remove_string", type=str, help="String to remove from filenames and folder names")
    parser.add_argument("-c", "--compress_spaces", action="store_true", help="Compress multiple spaces into one")
    parser.add_argument("--dry", action="store_true", help="Dry run. Do not actually rename anything.")
    parser.add_argument("-D", "--no_dirs", action="store_true", help="Don't rename directories.")
    parser.add_argument("-F", "--no_files", action="store_true", help="Don't rename files.")
    parser.add_argument("-i", "--case_insensitive", action="store_true", help="Perform case-insensitive compare and replace.")
    parser.add_argument("-v", "--verbose", action="store_true", help="Print what is being renamed.")

    args = parser.parse_args()

    rename_files_in_directory(args.dir, args.remove_string, args.compress_spaces, args.dry, args.verbose, args.no_dirs, args.no_files, args.case_insensitive)

if __name__ == "__main__":
    main()

Problem:

When you try running your command using pyinvoke’s invoke.run(), the native command returns a non-zero exit code making your script fail instead of just continuing (e.g. to evaluate the output):

Traceback (most recent call last):
File "/home/user/myscript.py", line 7, in <module>
result = invoke.run("mycommand", out_stream=outStream)
# ...
File "/usr/local/lib/python3.10/dist-packages/invoke/runners.py", line 518, in _finish
raise UnexpectedExit(result)
invoke.exceptions.UnexpectedExit: Encountered a bad command exit code!

Solution:

Use warn=True as an argument to invoke.run()

This will tell invoke to just warn you about the command exiting with non-zero exit status instead of throwing an UnexpectedExit

In our previous example How to capture stdout to string using invoke.run() in Python we showcased how to use invoke.run(..., out_stream=...) to capture stdout to a StringIO which can then be converted to a string.

Similarly, we can pass a string to the sub-process stdin by first converting it into a StringIO and then using invoke.run(..., in_stream=...).

stdin_str = "abc123" # what we want to pass to stdint

stdin_io = StringIO(stdin_str)
result = invoke.run("myexecutable", in_stream=stdin_in)
# ...

This example showcases how to capture stdout to a string instead of printing it to system stdout:

The basic trick is to initialize a StringIO (a file-like object where pyinvoke can write the stdout from the sub-process to) and pass it to the out_stream argument of invoke.run()

import invoke
from io import StringIO

outStream = StringIO()
result = invoke.run("wg genkey", out_stream=outStream)
stdout_str = outStream.getvalue()
print("Result: ", stdout_str)

Whereas Python’s os.listdir("mydirectory") will list both sub-files and sub-directories, you can use this simple list comprehension:

parent_folder = "myfolder"
sub_directories = [
    filename for filename in os.listdir(parent_folder)
    if os.path.isdir(os.path.join(parent_folder, filename))
]

which will only list sub-directories of the parent_fodler

You can just pass a io.BytesIO() to plt.savefig() as first parameter. Note that format defaults to "png", but best practice is to explicitly specify it.

# Save plot to BytesIO
bio = io.BytesIO()
plt.savefig(bio, dpi=250, format="png")

Full example:

#!/usr/bin/env python3
import numpy as np
import io
import matplotlib.pyplot as plt

# Generate data
x = np.linspace(0, 2*np.pi, 100)
y = np.sin(x)

# Plot data
plt.plot(x, y)

# Save plot to BytesIO
bio = io.BytesIO()
plt.savefig(bio, dpi=250, format="png")

# Cleanup plot
plt.close(plt.gcf())
plt.clf()

# Write BytesIO to file
with open("plot.png", "wb") as f:
    f.write(bio.getvalue())

Output plot.png:

I recommend using the configparser library which comes with Python for parsing INI files. However, if you need a highly customizable parser, this code is a good place to start:

def read_ini_config(filename):
    config = {}
    current_section = None

    with open(filename, 'r', encoding="utf-8") as file:
        for line in file:
            line = line.strip()

            # Skip empty lines and comments
            if not line or line.startswith(';') or line.startswith('#'):
                continue

            # Check if it's a section header
            if line.startswith('[') and line.endswith(']'):
                current_section = line[1:-1]
                config[current_section] = {}
            else:
                # Parse key-value pairs
                key, value = line.split('=', 1)
                key = key.strip()
                value = value.strip()
                config[current_section][key] = value

    return config

Problem:

You want to ignore a warning during a unit test using code such as

@pytest.mark.filterwarnings("divide by zero encountered in log10")
def test_auto_suffix_1d(self):
   # ...

but you see an error message such as

ERROR: while parsing the following warning configuration:

  divide by zero encountered in log10

This error occurred:

invalid action: 'divide by zero encountered in log10'

Solution:

In the argument to @pytest.mark.filterwarnings(...) you forgot the ignore: prefix:

Instead of

@pytest.mark.filterwarnings("divide by zero encountered in log10")

write

@pytest.mark.filterwarnings("ignore: divide by zero encountered in log10")

Problem:

You have a unit test case such as this one from UliEngineering

def test_ratio_to_dB_infinite(self):
    self.assertEqual(-np.inf, ratio_to_dB(0))
    self.assertEqual(-np.inf, ratio_to_dB(0, factor=dBFactor.Power))
    self.assertEqual(-np.inf, ratio_to_dB(-5))
    self.assertEqual(-np.inf, ratio_to_dB(-5, factor=dBFactor.Power))

which outputs a warning:

tests/Math/TestDecibel.py::TestDecibel::test_ratio_to_dB_infinite
  /home/uli/UliEngineering/UliEngineering/Math/Decibel.py:29: RuntimeWarning: divide by zero encountered in log10
    return np.log10(v)

Solution:

You can ignore that specific warning for this specific test case by using a pytest annotation:

import pytest

@pytest.mark.filterwarnings("ignore:divide by zero encountered in log10")
def test_ratio_to_dB_infinite(self):
    self.assertEqual(-np.inf, ratio_to_dB(0))
    self.assertEqual(-np.inf, ratio_to_dB(0, factor=dBFactor.Power))
    self.assertEqual(-np.inf, ratio_to_dB(-5))
    self.assertEqual(-np.inf, ratio_to_dB(-5, factor=dBFactor.Power))

Other warnings will still be emitted.

After, for example, re-encoding your videos, you often have a bunch of files in the same directory named e.g. MyMovie.avi and MyMovie.avi.mkv.

The following script will sort all files with the same prefix (MyMovie) into a directory called like the prefix. This allows easy sorting and comparing of those files.

Other files will not be touched. The operation is performed non-recursively and no files will be overwritten.

Note that the list of file extensions is hardcoded and double extensions such as .avi.mkv are also treated like standard extensions. The extension check is performed using .endswith()

#!/usr/bin/env python3
import argparse
import os
from collections import defaultdict

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("directory", help="path to the directory")
    args = parser.parse_args()

    directory = args.directory

    extensions = [
        ".avi",
        ".mkv",
        ".mp4"
    ]

    # Add extensions such as ".avi.mp4" at the front
    # so we check them first
    extensions = [e1 + e2 for e1 in extensions for e2 in extensions] + extensions

    files_by_prefix = defaultdict(list)
    for file in os.listdir(directory):
        # Split extension
        for potential_extension in extensions:
            if file.endswith(potential_extension):
                file_prefix = file[:-len(potential_extension)]
                files_by_prefix[file_prefix].append(file)

    # Ignore prefixes with only one file
    files_by_prefix = {k: v for k, v in files_by_prefix.items() if len(v) > 1}

    # Create directory for every prefix and move all the files into it
    for prefix, files in files_by_prefix.items():
        os.makedirs(os.path.join(directory, prefix), exist_ok=True)
        # Move file using if destination doesn't exist
        for file in files:
            print(f"{file} -> {prefix}/{file}")
            os.rename(os.path.join(directory, file), os.path.join(directory, prefix, file))