mp_pool

mp_poll is an example of Python multiprocessing using a process Pool.  Multiprocessing using a process Pool is a very simple way to implement parallel processing to increase performance.

Information and software on this web site are provided under the Granite Mountain Informatics LLC End User License Agreement (EULA).

Send questions or comments to GraniteMountainInformaticsLLC@gmail.com.

"""

=pod

=head1 NAME

mp_pool.py

=head1 VERSION

202302190559

=head1 SYNOPSIS

python3 mp_pool.py

=head1 DESCRIPTION

This is an example of Python multiprocessing using a process Pool.

=cut

"""

# ######################################################################

# IMPORT CORE MODULES

# ######################################################################

import datetime

import glob

import multiprocessing

from multiprocessing import Pool

import os

import random

from random import random

import re

import sys

import time

from time import sleep

# ######################################################################

# IMPORT CUSTOM MODULES

# ######################################################################

sys.path.insert(0, os.environ['fp_gmi_pylib'])

import gmi_datetime

import gmi_debug

import gmi_file

import gmi_string

import gmi_tdapi

# ######################################################################

# DECLARE GLOBAL VARIABLES

# ######################################################################

# ######################################################################

# DECLARE FUNCTIONS

# ######################################################################

"""

=pod

=head1 FUNCTIONS

=cut

"""

"""

=pod

=head2 mp_function(mp_args)

Execute the multiprocess function with the specified argument.

=cut

"""

def mp_function(mp_args):

# get the arguments

i, seconds = mp_args

# get the process id

pid = multiprocessing.current_process().pid

print(f"# [{pid}] mp_function({mp_args})", flush=True)

sleep(seconds)

return f"[{pid}] mp_function({mp_args}) slept for {seconds} seconds"

# ######################################################################

# MAIN

# ######################################################################

"""

The following "if __name__ == '__main__':" statement is a conditional

block explicitly identifies code to execute when your file is executed

as a script but not if the file is imported as a module.

"""

if __name__ == '__main__':

# print the command line

gmi_debug.print_sys_argv()

# print the cpu count

print(f"# CPU Count = {multiprocessing.cpu_count()}")

# set the mulitprocessing iteration argument

mp_args = []

for i in range(10):

mp_args.append([i, int(10 * random())])

# create the process pool

with Pool() as pool:

# iterate the mulitiprocess function over the arguments

for result in pool.map(mp_function, mp_args):

print(f"# {result}", flush=True)

"""

=pod

=head1 REFERENCE

=head1 REQUIRES

=head1 INSTALLATION

=head1 SEE ALSO

=head1 AUTHOR

Granite Mountain Informatics LLC

=head1 COPYRIGHT

Copyright 2023 Granite Mountain Informatics LLC

All Rights Reserved

=head1 PROPRIETARY SOFTWARE LICENSE

=head1 CHANGE HISTORY

=head1 TO DO

=cut

"""

The mp_pool.py program can be executed using the following commands in a linux shell script:

clear

fs_pypod='/Users/michaelhackerott/Documents/Granite Mountain Informatics LLC/Python/pypod/202302151410/pypod.py'

#echo "fs_pypod='${fs_pypod}'"

fp_pylib=`pwd`

#echo "fp_pylib='${fp_pylib}'"

function pypod()

{

rm -rf "${fp_pylib}/${1}.html"

python3 "${fs_pypod}" "${fp_pylib}/${1}"

open -a 'Google Chrome' "${fp_pylib}/${1}.html"

}

pypod mp_pool.py

export fp_gmi_pylib="/Users/michaelhackerott/Documents/Granite Mountain Informatics LLC/Python/gmi_pylib/202302151416"

python3 mp_pool.py

The output of the executed program:

% ./mp_pool.sh

/Users/michaelhackerott/Documents/Granite Mountain Informatics LLC/Python/pypod/202302151410/pypod.py /Users/michaelhackerott/Documents/Granite Mountain Informatics LLC/Python/mp_pool/202302190559/mp_pool.py

mp_pool.py

# CPU Count = 8

# [9144] mp_function([0, 9])

# [9146] mp_function([1, 1])

# [9145] mp_function([2, 6])

# [9147] mp_function([3, 2])

# [9148] mp_function([4, 5])

# [9149] mp_function([5, 0])

# [9149] mp_function([6, 5])

# [9151] mp_function([7, 5])

# [9150] mp_function([8, 3])

# [9146] mp_function([9, 9])

# [9144] mp_function([0, 9]) slept for 9 seconds

# [9146] mp_function([1, 1]) slept for 1 seconds

# [9145] mp_function([2, 6]) slept for 6 seconds

# [9147] mp_function([3, 2]) slept for 2 seconds

# [9148] mp_function([4, 5]) slept for 5 seconds

# [9149] mp_function([5, 0]) slept for 0 seconds

# [9149] mp_function([6, 5]) slept for 5 seconds

# [9151] mp_function([7, 5]) slept for 5 seconds

# [9150] mp_function([8, 3]) slept for 3 seconds

# [9146] mp_function([9, 9]) slept for 9 seconds

The mp_pool.py documentation: