In [1]:
%autosave 60
import numpy as np
from threading import Thread
from functools import partial
import multiprocessing
import multiprocessing.pool

Autosaving every 60 seconds
In [2]:
N_CPU = multiprocessing.cpu_count()
REPEAT = 16 * N_CPU
In [3]:
from time import sleep

def f():
    sleep(3)
    print('hello')
    
t = Thread(target=f)
t.start()
print('world')
t.join()
print('!')

world
hello
!
In [4]:
A = np.random.normal(0, 1, (64, 64, 5, 5))

%timeit np.linalg.eigvals(A)

35.7 ms ± 597 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
In [5]:
%%timeit A = np.random.normal(0, 1, (64, 64, 5, 5))
with multiprocessing.pool.ThreadPool() as pool:
    pool.map(lambda _: np.linalg.eigvals(A), range(REPEAT))

690 ms ± 34.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [6]:
class Eigen:
    def __init__(self):
        self.A = np.random.normal(0, 1, (64, 64, 5, 5))
    
    def __call__(self, x):
        A = self.A.copy()
        A[0, 0] += x
        vals = np.linalg.eigvals(A)
        return np.linalg.norm(vals)

f = Eigen()
In [7]:
%%timeit
with multiprocessing.pool.ThreadPool() as pool:
    pool.map(f, range(REPEAT))

1.42 s ± 64.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [8]:
%timeit [f(x) for x in range(REPEAT)]

4.75 s ± 143 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [9]:
def golden_ratio(n):
    s = 1.0
    for _ in range(n):
        s = 1.0 + 1.0 / s
    return s
In [10]:
%timeit [golden_ratio(100_000) for _ in range(REPEAT)]

882 ms ± 32.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [11]:
%%timeit
with multiprocessing.pool.ThreadPool() as pool:
    pool.map(lambda _: golden_ratio(100_000), range(REPEAT))

930 ms ± 31.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [12]:
# GIL
In [13]:
# GIL отпущен во время всех I/O операций

# with open()...

# GIL отпущен в некоторых функциях некоторых "нативных" модулей
# типа numpy, scipy, ...
In [18]:
%%timeit
with multiprocessing.Pool() as pool:
    pool.map(golden_ratio, [100_000] * REPEAT)

262 ms ± 2.34 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [23]:
%%timeit a = np.random.rand(10_000_000)
for _ in range(REPEAT):
    np.sum(a)

1.49 s ± 6.08 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [24]:
%%timeit a = np.random.rand(10_000_000)
with multiprocessing.Pool() as pool:
    pool.map(np.sum, [a] * REPEAT)

8.37 s ± 104 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [25]:
import pickle
In [28]:
x = 123
pickle.dumps(x)
# pickle.dumps(lambda x: x**2) - ошибка
Out[28]:
b'\x80\x03K{.'
In [29]:
from functools import partial
In [30]:
with multiprocessing.Pool() as pool:
    result = pool.map(np.random.rand, [1] * N_CPU)
print(result)

[array([0.89921603]), array([0.89921603]), array([0.89921603]), array([0.89921603]), array([0.89921603]), array([0.89921603]), array([0.89921603]), array([0.89921603])]
In [31]:
[np.random.rand() for _ in range(N_CPU)]
Out[31]:
[0.8992160348470516,
 0.9551956229774841,
 0.07824628133895539,
 0.823248685458748,
 0.13479980996708463,
 0.3900273918893963,
 0.07412678098979342,
 0.8652795249825378]
In [33]:
a = []

def f(x):
    a.append(x)
    print(a)
    

with multiprocessing.Pool() as pool:
    pool.map(f, range(REPEAT))
    
print(a)

[0]
[4]
[8]
[12]
[8, 9]
[24]
[16]
[20]
[28]
[0, 1]
[4, 5]
[12, 13]
[20, 21]
[12, 13, 14]
[8, 9, 10]
[4, 5, 6]
[24, 25]
[16, 17]
[20, 21, 22]
[4, 5, 6, 7]
[12, 13, 14, 15]
[24, 25, 26]
[8, 9, 10, 11]
[0, 1, 2]
[16, 17, 18]
[20, 21, 22, 23]
[16, 17, 18, 19]
[4, 5, 6, 7, 36]
[24, 25, 26, 27]
[0, 1, 2, 3]
[12, 13, 14, 15, 32]
[16, 17, 18, 19, 48]
[12, 13, 14, 15, 32, 33]
[20, 21, 22, 23, 44]
[16, 17, 18, 19, 48, 49]
[12, 13, 14, 15, 32, 33, 34]
[8, 9, 10, 11, 40]
[24, 25, 26, 27, 56]
[4, 5, 6, 7, 36, 37]
[0, 1, 2, 3, 52]
[28, 29, 30]
[24, 25, 26, 27, 56, 57]
[20, 21, 22, 23, 44, 45]
[4, 5, 6, 7, 36, 37, 38]
[28, 29]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61]
[16, 17, 18, 19, 48, 49, 50]
[8, 9, 10, 11, 40, 41]
[20, 21, 22, 23, 44, 45, 46]
[12, 13, 14, 15, 32, 33, 34, 35]
[0, 1, 2, 3, 52, 53]
[24, 25, 26, 27, 56, 57, 58]
[4, 5, 6, 7, 36, 37, 38, 39]
[16, 17, 18, 19, 48, 49, 50, 51]
[20, 21, 22, 23, 44, 45, 46, 47]
[8, 9, 10, 11, 40, 41, 42]
[4, 5, 6, 7, 36, 37, 38, 39, 60]
[12, 13, 14, 15, 32, 33, 34, 35, 64]
[20, 21, 22, 23, 44, 45, 46, 47, 68]
[28, 29, 30, 31]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65]
[8, 9, 10, 11, 40, 41, 42, 43]
[16, 17, 18, 19, 48, 49, 50, 51, 72]
[16, 17, 18, 19, 48, 49, 50, 51, 72, 73]
[28, 29, 30, 31, 76]
[0, 1, 2, 3, 52, 53, 54]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65, 66]
[20, 21, 22, 23, 44, 45, 46, 47, 68, 69]
[8, 9, 10, 11, 40, 41, 42, 43, 80]
[16, 17, 18, 19, 48, 49, 50, 51, 72, 73, 74]
[28, 29, 30, 31, 76, 77]
[24, 25, 26, 27, 56, 57, 58, 59]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65, 66, 67]
[8, 9, 10, 11, 40, 41, 42, 43, 80, 81]
[0, 1, 2, 3, 52, 53, 54, 55]
[24, 25, 26, 27, 56, 57, 58, 59, 84]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61, 62]
[8, 9, 10, 11, 40, 41, 42, 43, 80, 81, 82]
[20, 21, 22, 23, 44, 45, 46, 47, 68, 69, 70]
[24, 25, 26, 27, 56, 57, 58, 59, 84, 85]
[28, 29, 30, 31, 76, 77, 78]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61, 62, 63]
[8, 9, 10, 11, 40, 41, 42, 43, 80, 81, 82, 83]
[24, 25, 26, 27, 56, 57, 58, 59, 84, 85, 86]
[16, 17, 18, 19, 48, 49, 50, 51, 72, 73, 74, 75]
[8, 9, 10, 11, 40, 41, 42, 43, 80, 81, 82, 83, 96]
[28, 29, 30, 31, 76, 77, 78, 79]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61, 62, 63, 92]
[20, 21, 22, 23, 44, 45, 46, 47, 68, 69, 70, 71]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65, 66, 67, 100]
[24, 25, 26, 27, 56, 57, 58, 59, 84, 85, 86, 87]
[8, 9, 10, 11, 40, 41, 42, 43, 80, 81, 82, 83, 96, 97]
[20, 21, 22, 23, 44, 45, 46, 47, 68, 69, 70, 71, 108]
[16, 17, 18, 19, 48, 49, 50, 51, 72, 73, 74, 75, 104]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61, 62, 63, 92, 93]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65, 66, 67, 100, 101]
[28, 29, 30, 31, 76, 77, 78, 79, 112]
[20, 21, 22, 23, 44, 45, 46, 47, 68, 69, 70, 71, 108, 109]
[16, 17, 18, 19, 48, 49, 50, 51, 72, 73, 74, 75, 104, 105]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65, 66, 67, 100, 101, 102]
[0, 1, 2, 3, 52, 53, 54, 55, 88]
[8, 9, 10, 11, 40, 41, 42, 43, 80, 81, 82, 83, 96, 97, 98]
[24, 25, 26, 27, 56, 57, 58, 59, 84, 85, 86, 87, 116]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61, 62, 63, 92, 93, 94]
[16, 17, 18, 19, 48, 49, 50, 51, 72, 73, 74, 75, 104, 105, 106]
[20, 21, 22, 23, 44, 45, 46, 47, 68, 69, 70, 71, 108, 109, 110]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65, 66, 67, 100, 101, 102, 103]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61, 62, 63, 92, 93, 94, 95]
[0, 1, 2, 3, 52, 53, 54, 55, 88, 89]
[20, 21, 22, 23, 44, 45, 46, 47, 68, 69, 70, 71, 108, 109, 110, 111]
[16, 17, 18, 19, 48, 49, 50, 51, 72, 73, 74, 75, 104, 105, 106, 107]
[24, 25, 26, 27, 56, 57, 58, 59, 84, 85, 86, 87, 116, 117]
[8, 9, 10, 11, 40, 41, 42, 43, 80, 81, 82, 83, 96, 97, 98, 99]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61, 62, 63, 92, 93, 94, 95, 124]
[24, 25, 26, 27, 56, 57, 58, 59, 84, 85, 86, 87, 116, 117, 118]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61, 62, 63, 92, 93, 94, 95, 124, 125]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65, 66, 67, 100, 101, 102, 103, 120]
[0, 1, 2, 3, 52, 53, 54, 55, 88, 89, 90]
[24, 25, 26, 27, 56, 57, 58, 59, 84, 85, 86, 87, 116, 117, 118, 119]
[0, 1, 2, 3, 52, 53, 54, 55, 88, 89, 90, 91]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61, 62, 63, 92, 93, 94, 95, 124, 125, 126]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65, 66, 67, 100, 101, 102, 103, 120, 121]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65, 66, 67, 100, 101, 102, 103, 120, 121, 122]
[4, 5, 6, 7, 36, 37, 38, 39, 60, 61, 62, 63, 92, 93, 94, 95, 124, 125, 126, 127]
[12, 13, 14, 15, 32, 33, 34, 35, 64, 65, 66, 67, 100, 101, 102, 103, 120, 121, 122, 123]
[28, 29, 30, 31, 76, 77, 78, 79, 112, 113]
[28, 29, 30, 31, 76, 77, 78, 79, 112, 113, 114]
[28, 29, 30, 31, 76, 77, 78, 79, 112, 113, 114, 115]
[]
In [34]:
# cloudpickle
# joblib
# dask
In [ ]: