In [11]:
%autosave 20

import pickle
from functools import partial
from multiprocessing import cpu_count
from multiprocessing import Pool as ProcessPool
from multiprocessing.pool import ThreadPool
from threading import Thread
from queue import Queue

import numpy as np
Autosaving every 20 seconds
In [2]:
cpus = cpu_count()
repeat = 8*cpus
In [3]:
A = np.random.rand(64, 64, 5, 5)
def f(x):
    return np.linalg.eigvals(A + x)

# def f(x):
#     return np.log(np.random.rand(100_000)**2 + x*500 + 1)**np.pi

# A = np.random.rand(100_000, 2)
# b = np.random.rand(100_000)
# def f(x):
#     return np.linalg.lstsq(A, b, rcond=None)

# a = list(range(1_000_000))

# def f(x, y):
#     return sum(a) * x + y
In [4]:
%timeit f(0)
41 ms ± 1.18 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
In [5]:
%%timeit
for i in range(repeat):
    f(i)
2.34 s ± 5.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [6]:
%%timeit
threads = tuple(Thread(target=f, daemon=False, args=(i,))
                for i in range(repeat))
for t in threads:
    t.start()
for t in threads:
    t.join()
331 ms ± 7.87 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [7]:
%%timeit
with ThreadPool(processes=cpus) as pool:
    result = pool.map(f, range(repeat))
389 ms ± 34.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [8]:
%%timeit
with ProcessPool(processes=cpus) as pool:
#     result = pool.map(partial(f, 2), range(repeat))
    result = pool.map(f, range(repeat))
437 ms ± 1.65 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [9]:
b = pickle.dumps([1,2])
pickle.loads(b)
Out[9]:
[1, 2]
In [16]:
q = Queue()
q.put(1)
q.get(timeout=1)
q.get()
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-16-9a144bc03be6> in <module>()
      2 q.put(1)
      3 q.get(timeout=1)
----> 4 q.get()

/opt/conda/lib/python3.6/queue.py in get(self, block, timeout)
    162             elif timeout is None:
    163                 while not self._qsize():
--> 164                     self.not_empty.wait()
    165             elif timeout < 0:
    166                 raise ValueError("'timeout' must be a non-negative number")

/opt/conda/lib/python3.6/threading.py in wait(self, timeout)
    293         try:    # restore state no matter what (e.g., KeyboardInterrupt)
    294             if timeout is None:
--> 295                 waiter.acquire()
    296                 gotit = True
    297             else:

KeyboardInterrupt: