Alle PubMed-Baseline-Dateien parallel mit Python parsen

pip install git+git://github.com/titipata/pubmed_parser.git six numpy tqdm

#!/usr/bin/env python3
import pubmed_parser as pp
import glob
import os
from collections import Counter
import concurrent.futures
from tqdm import tqdm

# Source: https://techoverflow.net/2017/05/18/how-to-use-concurrent-futures-map-with-a-tqdm-progress-bar/
def tqdm_parallel_map(executor, fn, *iterables, **kwargs):
    """
    Entspricht executor.map(fn, *iterables),
    zeigt aber eine tqdm-basierte Fortschrittsanzeige.

    Unterstützt kein timeout oder chunksize, da executor.submit intern verwendet wird

    **kwargs wird an tqdm übergeben.
    """
    futures_list = []
    for iterable in iterables:
        futures_list += [executor.submit(fn, i) for i in iterable]
    for f in tqdm(concurrent.futures.as_completed(futures_list), total=len(futures_list), **kwargs):
        yield f.result()

def parse_and_process_file(filename):
    """
    This function contains our parsing code. Normally you only modify this function.
    """
    # Autoren und Referenzen für dieses Beispiel nicht parsen, da wir sie nicht benötigen
    dat = pp.parse_medline_xml(filename, author_list=False, reference_list=False)

    # For this example, we'll build a set of count of all MeSH IDs in this file
    ctr = Counter()
    for entry in dat:
        terms = [term.partition(":")[0].strip() for term in entry["mesh_terms"].split(";")]
        for term in terms:
            ctr[term] += 1
    return filename, ctr


if __name__ == "__main__":
    # Find all pubmed files in the current directory
    all_filenames = glob.glob("pubmed*.xml.gz")
    # For some workloads you might want to use a ThreadPoolExecutor,
    # aber ein ProcessPoolExecutor ist ein guter Standard
    executor = concurrent.futures.ProcessPoolExecutor(os.cpu_count())
    # Iterate results as they come in (the order is not the same as in the input!)
    for filename, ctr in tqdm_parallel_map(executor, parse_and_process_file, all_filenames):
        # NOTE: If you print() here, this might interfere with the progress bar,
        # aber das akzeptieren wir hier, da es nur ein Beispiel ist
        print(filename, ctr)