multiple objects in pickle file

  • Last Update :
  • Techknowledgy :

First, you need not store the number of items you pickled separately if you stop loading when you hit the end of the file:

def loadall(filename):
   with open(filename, "rb") as f:
   while True:
   try:
   yield pickle.load(f)
except EOFError:
   break

items = loadall(myfilename)

Using a list, tuple, or dict is by far the most common way to do this:

import pickle
PIK = "pickle.dat"

data = ["A", "b", "C", "d"]
with open(PIK, "wb") as f:
   pickle.dump(data, f)
with open(PIK, "rb") as f:
   print pickle.load(f)

That prints:

['A', 'b', 'C', 'd']

However, a pickle file can contain any number of pickles. Here's code producing the same output. But note that it's harder to write and to understand:

with open(PIK, "wb") as f:
   pickle.dump(len(data), f)
for value in data:
   pickle.dump(value, f)
data2 = []
with open(PIK, "rb") as f:
   for _ in range(pickle.load(f)):
   data2.append(pickle.load(f))
print data2

Try this:

import pickle

file = open('test.pkl', 'wb')
obj_1 = ['test_1', {
   'ability',
   'mobility'
}]
obj_2 = ['test_2', {
   'ability',
   'mobility'
}]
obj_3 = ['test_3', {
   'ability',
   'mobility'
}]

pickle.dump(obj_1, file)
pickle.dump(obj_2, file)
pickle.dump(obj_3, file)

file.close()

file = open('test.pkl', 'rb')
obj_1 = pickle.load(file)
obj_2 = pickle.load(file)
obj_3 = pickle.load(file)
print(obj_1)
print(obj_2)
print(obj_3)
file.close()

You can run a loop (as the accepted answer shows) to keep unpickling rows until you reach the end-of-file (at which point an EOFError is raised).

data = []
with open("data.pickle", "rb") as f:
   while True:
   try:
   data.append(pickle.load(f))
except EOFError:
   break

Minimal Verifiable Example

import pickle

# Dumping step
data = [{
   'a': 1
}, {
   'b': 2
}]
with open('test.pkl', 'wb') as f:
   for d in data:
   pickle.dump(d, f)

# Loading step
data2 = []
with open('test.pkl', 'rb') as f:
   while True:
   try:
   data2.append(pickle.load(f))
except EOFError:
   break

data2
#[{
   'a': 1
}, {
   'b': 2
}]

data == data2
# True

Of course, this is under the assumption that your objects have to be pickled individually. You can also store your data as a single list of object, then use a single pickle/unpickle call (no need for loops).

data = [{
   'a': 1
}, {
   'b': 2
}] # list of dicts as an example
with open('test.pkl', 'wb') as f:
   pickle.dump(data, f)

with open('test.pkl', 'rb') as f:
   data2 = pickle.load(f)

data2
#[{
   'a': 1
}, {
   'b': 2
}]

I will give an object-oriented demo using pickle to store and restore one or multi object:

class Worker(object):

def __init__(self, name, addr):
self.name = name
self.addr = addr

def __str__(self):
string = u'[<Worker> name:%s addr:%s]' %(self.name, self.addr)
   return string

   # output one item
   with open('testfile.bin', 'wb') as f:
   w1 = Worker('tom1', 'China')
   pickle.dump(w1, f)

   # input one item
   with open('testfile.bin', 'rb') as f:
   w1_restore = pickle.load(f)
   print 'item: %s' %w1_restore

   # output multi items
   with open('testfile.bin', 'wb') as f:
   w1 = Worker('tom2', 'China')
   w2 = Worker('tom3', 'China')
   pickle.dump([w1, w2], f)

   # input multi items
   with open('testfile.bin', 'rb') as f:
   w_list = pickle.load(f)

   for w in w_list:
   print 'item-list: %s' %w

output:

item: [<Worker> name:tom1 addr:China]
   item-list: [<Worker> name:tom2 addr:China]
      item-list: [<Worker> name:tom3 addr:China]

It's easy if you use klepto, which gives you the ability to transparently store objects in files or databases. It uses a dict API, and allows you to dump and/or load specific entries from an archive (in the case below, serialized objects stored one entry per file in a directory called scores).

>>>
import klepto
   >>>
   scores = klepto.archives.dir_archive('scores', serialized = True) >>>
   scores['Guido'] = 69 >>>
   scores['Fernando'] = 42 >>>
   scores['Polly'] = 101 >>>
   scores.dump() >>>
   # access the archive, and load only one >>>
   results = klepto.archives.dir_archive('scores', serialized = True) >>>
   results.load('Polly') >>>
   results
dir_archive('scores', {
      'Polly': 101
   }, cached = True) >>>
   results['Polly']
101
   >>>
   # load all the scores >>>
   results.load() >>>
   results['Guido']
69
   >>>

Suggestion : 2

Post date April 15, 2022 ,© 2022 The Web Dev

For instance, we write

def loadall(filename):
   with open(filename, "rb") as f:
   while True:
   try:
   yield pickle.load(f)
except EOFError:
   break

items = loadall(my_filename)

Suggestion : 3

If you want to serialize and deserialize Python objects you might have considered using the Python Pickle module.,The Python Pickle module is used to perform serialization and deserialization of Python objects.,Python Pickle Example,With the pickle module you can save different types of Python objects.

Let’s see an example of how you can use the pickle module to serialize a Python list.

>>>
import pickle
   >>>
   animals = ['tiger', 'lion', 'giraffe'] >>>
   pickle.dumps(animals) b '\x80\x04\x95\x1e\x00\x00\x00\x00\x00\x00\x00]\x94(\x8c\x05tiger\x94\x8c\x04lion\x94\x8c\x07giraffe\x94e.'

Now, we will store the pickled string in a variable and use the loads() function to convert the bytes string back to our original list.

>>> pickled_animals = pickle.dumps(animals) >>>
   unpickled_animals = pickle.loads(pickled_animals) >>>
   print(unpickled_animals)['tiger', 'lion', 'giraffe']

Let’s use the dumps() function to pickle a Python dictionary.

>>> animals = {
      'tiger': 23,
      'lion': 45,
      'giraffe': 67
   } >>>
   pickled_animals = pickle.dumps(animals) >>>
   print(pickled_animals)
b '\x80\x04\x95$\x00\x00\x00\x00\x00\x00\x00}\x94(\x8c\x05tiger\x94K\x17\x8c\x04lion\x94K-\x8c\x07giraffe\x94KCu.'

Firstly, we will open a file in binary mode using the Python open function, store the pickled dictionary in the file and close the file.

>>>
import pickle
   >>>
   animals = {
      'tiger': 23,
      'lion': 45,
      'giraffe': 67
   } >>>
   f = open('data.pickle', 'wb') >>>
   pickle.dump(animals, f) >>>
   f.close()

If you look at the content of the data.pickle file with a text editor you will see data in binary format.

ۥ
$
}”(Œ tiger” K Œ lion” K - Œ giraffe” KCu.

Suggestion : 4

The pickle module keeps track of the objects it has already serialized, so that later references to the same object won’t be serialized again. marshal doesn’t do this.,marshal cannot be used to serialize user-defined classes and their instances. pickle can save and restore class instances transparently, however the class definition must be importable and live in the same module as when the object was stored.,To unpickle external objects, the unpickler must have a custom persistent_load() method that takes a persistent ID object and returns the referenced object.,For the benefit of object persistence, the pickle module supports the notion of a reference to an object outside the pickled data stream. Such objects are referenced by a persistent ID, which should be either a string of alphanumeric characters (for protocol 0) 5 or just an arbitrary object (for any newer protocol).

class Foo:
   attr = 'A class attribute'

picklestring = pickle.dumps(Foo)
def save(obj):
   return (obj.__class__, obj.__dict__)

def restore(cls, attributes):
   obj = cls.__new__(cls)
obj.__dict__.update(attributes)
return obj
f = io.BytesIO()
p = pickle.Pickler(f)
p.dispatch_table = copyreg.dispatch_table.copy()
p.dispatch_table[SomeClass] = reduce_SomeClass
class MyPickler(pickle.Pickler):
   dispatch_table = copyreg.dispatch_table.copy()
dispatch_table[SomeClass] = reduce_SomeClass
f = io.BytesIO()
p = MyPickler(f)
copyreg.pickle(SomeClass, reduce_SomeClass)
f = io.BytesIO()
p = pickle.Pickler(f)

Suggestion : 5

pickle.dumpwill append to the end of the file, so you can call it multiple times to write multiple values. pickle.loadwill read only enough from the file to get the first value, leaving the filehandle open and pointed at the start of the next object in the file. , 6 days ago pickle.dump will append to the end of the file, so you can call it multiple times to write multiple values. pickle.load will read only enough from the file to get the first value, leaving the filehandle open and pointed at the start of the next object in the file. The second call will then read the second object, and leave the file pointer at ... , 1 week ago pickle.dump will append to the end of the file, so you can call it multiple times to write multiple values. pickle.load will read only enough from the file to get the first value, leaving the filehandle open and pointed at the start of the next object in the file. The second call will then read the second object, and leave the file pointer at ... , 3 days ago Mar 16, 2013  · pickle.dump将追加到文件的末尾,因此您可以多次调用它来写入多个值。 pickle.load will read only enough from the file to get the first value, leaving the filehandle open and pointed at the start of the next object in the file. The second call will then read the second object, and leave the file pointer at the end of the file.


myob = cpickle.load(g1.readlines()[2])
myob = cpickle.load(g1.readlines()[2])
import pickle # write a file f = open("example", "w") pickle.dump(["hello", "world"], f) pickle.dump([2, 3], f) f.close() f = open("example", "r") value1 = pickle.load(f) value2 = pickle.load(f) f.close()
a = [1, 2] b = [3, 4] with open("tmp.pickle", "wb") as f: pickle.dump((a, b), f) with open("tmp.pickle", "rb") as f: a, b = pickle.load(f)

Suggestion : 6

Is this the most efficient JavaScript for comparing two arrays of objects and modifying properties?,I will give an object-oriented demo anycodings_python using pickle to store and restore one or anycodings_python multi object:,You can run a loop (as the accepted anycodings_python answer shows) to keep unpickling rows anycodings_python until you reach the end-of-file (at anycodings_python which point an EOFError is raised).,If you do this, you're responsible for anycodings_python knowing how many pickles are in the file anycodings_python you write out. The code above does that anycodings_python by pickling the number of list objects anycodings_python first.

First, you need not store the number of anycodings_python items you pickled separately if you stop anycodings_python loading when you hit the end of the anycodings_python file:

def loadall(filename):
   with open(filename, "rb") as f:
   while True:
   try:
   yield pickle.load(f)
except EOFError:
   break

items = loadall(myfilename)

Using a list, tuple, or dict is by far anycodings_python the most common way to do this:

import pickle
PIK = "pickle.dat"

data = ["A", "b", "C", "d"]
with open(PIK, "wb") as f:
   pickle.dump(data, f)
with open(PIK, "rb") as f:
   print pickle.load(f)

That prints:

['A', 'b', 'C', 'd']

However, a pickle file can contain any anycodings_python number of pickles. Here's code anycodings_python producing the same output. But note anycodings_python that it's harder to write and to anycodings_python understand:

with open(PIK, "wb") as f:
   pickle.dump(len(data), f)
for value in data:
   pickle.dump(value, f)
data2 = []
with open(PIK, "rb") as f:
   for _ in range(pickle.load(f)):
   data2.append(pickle.load(f))
print data2

Try this:

import pickle

file = open('test.pkl', 'wb')
obj_1 = ['test_1', {
   'ability',
   'mobility'
}]
obj_2 = ['test_2', {
   'ability',
   'mobility'
}]
obj_3 = ['test_3', {
   'ability',
   'mobility'
}]

pickle.dump(obj_1, file)
pickle.dump(obj_2, file)
pickle.dump(obj_3, file)

file.close()

file = open('test.pkl', 'rb')
obj_1 = pickle.load(file)
obj_2 = pickle.load(file)
obj_3 = pickle.load(file)
print(obj_1)
print(obj_2)
print(obj_3)
file.close()

You can run a loop (as the accepted anycodings_python answer shows) to keep unpickling rows anycodings_python until you reach the end-of-file (at anycodings_python which point an EOFError is raised).

data = []
with open("data.pickle", "rb") as f:
   while True:
   try:
   data.append(pickle.load(f))
except EOFError:
   break

Minimal Verifiable Example

import pickle

# Dumping step
data = [{
   'a': 1
}, {
   'b': 2
}]
with open('test.pkl', 'wb') as f:
   for d in data:
   pickle.dump(d, f)

# Loading step
data2 = []
with open('test.pkl', 'rb') as f:
   while True:
   try:
   data2.append(pickle.load(f))
except EOFError:
   break

data2
#[{
   'a': 1
}, {
   'b': 2
}]

data == data2
# True

Of course, this is under the assumption anycodings_python that your objects have to be pickled anycodings_python individually. You can also store your anycodings_python data as a single list of object, then anycodings_python use a single pickle/unpickle call (no anycodings_python need for loops).

data = [{
   'a': 1
}, {
   'b': 2
}] # list of dicts as an example
with open('test.pkl', 'wb') as f:
   pickle.dump(data, f)

with open('test.pkl', 'rb') as f:
   data2 = pickle.load(f)

data2
#[{
   'a': 1
}, {
   'b': 2
}]

I will give an object-oriented demo anycodings_python using pickle to store and restore one or anycodings_python multi object:

class Worker(object):

def __init__(self, name, addr):
self.name = name
self.addr = addr

def __str__(self):
string = u'[<Worker> name:%s addr:%s]' %(self.name, self.addr)
   return string

   # output one item
   with open('testfile.bin', 'wb') as f:
   w1 = Worker('tom1', 'China')
   pickle.dump(w1, f)

   # input one item
   with open('testfile.bin', 'rb') as f:
   w1_restore = pickle.load(f)
   print 'item: %s' %w1_restore

   # output multi items
   with open('testfile.bin', 'wb') as f:
   w1 = Worker('tom2', 'China')
   w2 = Worker('tom3', 'China')
   pickle.dump([w1, w2], f)

   # input multi items
   with open('testfile.bin', 'rb') as f:
   w_list = pickle.load(f)

   for w in w_list:
   print 'item-list: %s' %w

output:

item: [<Worker> name:tom1 addr:China]
   item-list: [<Worker> name:tom2 addr:China]
      item-list: [<Worker> name:tom3 addr:China]

It's easy if you use klepto, which gives anycodings_python you the ability to transparently store anycodings_python objects in files or databases. It uses anycodings_python a dict API, and allows you to dump anycodings_python and/or load specific entries from an anycodings_python archive (in the case below, serialized anycodings_python objects stored one entry per file in a anycodings_python directory called scores).

>>>
import klepto
   >>>
   scores = klepto.archives.dir_archive('scores', serialized = True) >>>
   scores['Guido'] = 69 >>>
   scores['Fernando'] = 42 >>>
   scores['Polly'] = 101 >>>
   scores.dump() >>>
   # access the archive, and load only one >>>
   results = klepto.archives.dir_archive('scores', serialized = True) >>>
   results.load('Polly') >>>
   results
dir_archive('scores', {
      'Polly': 101
   }, cached = True) >>>
   results['Polly']
101
   >>>
   # load all the scores >>>
   results.load() >>>
   results['Guido']
69
   >>>