[reportlab-users] Re: Generating huge reports and memory consumption
Robin Becker
reportlab-users@reportlab.com
Sat, 4 Jan 2003 14:32:15 +0000
In article <oq4r8q2as1.fsf@titan.progiciels-bpi.ca>, François Pinard <pinard@iro.umontreal.ca> writes
.....
following your and Martin v. Löwis' suggestions
I used code below which attempts to count everything and uses len, marshal, cPickle to try and size
everything.
I used python -v to run the test script to produce 400 & 500 pages.
The output does not show increases in gc visible objects.
The gc_check for 400 produces the same output as for 500 ie I see stability
in gc visible objects.
Even so I still see differences in the freed floats
(from the -v option cleanup output). Interestingly I see no floats
mentioned in the output of gc_check. I assume they're all in containers.
case 400
# cleanup floats: 4544 unfreed floats in 74 out of 74 blocks
case 500
# cleanup floats: 5544 unfreed floats in 90 out of 90 blocks
Is there any way to get more information about the 'unfreed' things
mentioned by the cleanup?
##########################################
import gc, marshal, cPickle
from types import InstanceType
def gc_check(quiet=0):
if quiet: return
print 'Number collected:',gc.collect(),'Length of garbage:',len(gc.garbage)
O = gc.get_objects()
print 'Number of objects:', len(O)
D = {}
L = {}
M = {}
P = {}
for o in O:
t = type(o)
if t is InstanceType:
name = '<class %s>' % o.__class__.__name__
if not D.has_key(name): D[name] = 1
else: D[name] += 1
else:
name = t.__name__
if not D.has_key(name): D[name] = 1
else: D[name] += 1
try:
L[name] = L.get(name,0)+len(o)
except:
pass
try:
M[name] = M.get(name,0)+len(marshal.dumps(o))
except:
pass
try:
P[name] = P.get(name,0)+len(cPickle.dumps(o))
except:
pass
K = D.keys()
K.sort()
print 'Objects:'
for k in K:
print '',k,D[k],L.get(k,'?'),M.get(k,'?'),P.get(k,'?')
from reportlab.lib.styles import ParagraphStyle
from reportlab.platypus import *
def main(n,quiet):
story = []
i = 0
for x in xrange(n):
i = i + 1
story.append(Paragraph(str(i),ParagraphStyle('normal')))
story.append(PageBreak())
if i % 100==0:
fn = "test_%04d.pdf" % i
SimpleDocTemplate(fn).build(story)
print 'Build',fn
story = []
gc_check(quiet)
if story:
fn = "test_%04d.pdf"%i
SimpleDocTemplate(fn).build(story)
print 'Build',fn
story = []
gc_check(quiet)
if __name__=='__main__':
import sys
quiet = '-q' in sys.argv
if quiet: sys.argv.remove('-q')
try:
n = int(sys.argv[1])
except:
n = 400
main(n,quiet)
gc_check(quiet)
##########################################
Last iterations of output.
Build test_0400.pdf
Number collected: 21 Length of garbage: 0
Number of objects: 6741
Objects:
<class ActionFlowable> 1 ? ? 93
<class CMYKColor> 2 ? ? 418
<class Color> 143 ? ? 17859
<class Encoding> 1 ? ? 3089
<class Font> 2 ? ? 17592
<class Logger> 1 ? ? ?
<class MemoryError> 1 ? ? 52
<class NoEncryption> 2 ? ? 100
<class PCMYKColor> 2 ? ? 420
<class PDFStreamFilterBase85Encode> 1 ? ? 65
<class PDFStreamFilterZCompress> 1 ? ? 62
<class ParaParser> 1 ? ? ?
<class Sequencer> 1 ? ? 255
<class TableStyle> 5 ? ? 1977
<class TypeFace> 2 ? ? 11888
<class WarnOnce> 2 ? ? 202
<class _Environ> 1 36 ? 2282
<class _Feature> 3 ? ? 438
<class _FrameBreak> 1 ? ? 89
<class _Helper> 1 ? ? 25
<class _Name2StandardEncodingMap> 1 7 ? 14599
<class _Printer> 3 ? ? 1108
<class _ThreadSafeCounter> 1 ? ? ?
class 167 ? 1 6231
dict 595 14011 305269 388256
frame 3 ? ? ?
function 1053 ? ? 9223
getset_descriptor 18 ? ? ?
instance method 19 ? ? ?
list 94 1442 12843 17399
member_descriptor 91 ? ? ?
method_descriptor 112 ? ? ?
module 77 ? ? ?
tuple 3897 21737 299169 331236
weakref 37 ? ? ?
wrapper_descriptor 399 ? ? ?
Build test_0500.pdf
Number collected: 21 Length of garbage: 0
Number of objects: 6741
Objects:
<class ActionFlowable> 1 ? ? 93
<class CMYKColor> 2 ? ? 418
<class Color> 143 ? ? 17859
<class Encoding> 1 ? ? 3089
<class Font> 2 ? ? 17592
<class Logger> 1 ? ? ?
<class MemoryError> 1 ? ? 52
<class NoEncryption> 2 ? ? 100
<class PCMYKColor> 2 ? ? 420
<class PDFStreamFilterBase85Encode> 1 ? ? 65
<class PDFStreamFilterZCompress> 1 ? ? 62
<class ParaParser> 1 ? ? ?
<class Sequencer> 1 ? ? 255
<class TableStyle> 5 ? ? 1977
<class TypeFace> 2 ? ? 11888
<class WarnOnce> 2 ? ? 202
<class _Environ> 1 36 ? 2282
<class _Feature> 3 ? ? 438
<class _FrameBreak> 1 ? ? 89
<class _Helper> 1 ? ? 25
<class _Name2StandardEncodingMap> 1 7 ? 14599
<class _Printer> 3 ? ? 1108
<class _ThreadSafeCounter> 1 ? ? ?
class 167 ? 1 6231
dict 595 14011 305269 388256
frame 3 ? ? ?
function 1053 ? ? 9223
getset_descriptor 18 ? ? ?
instance method 19 ? ? ?
list 94 1442 12843 17399
member_descriptor 91 ? ? ?
method_descriptor 112 ? ? ?
module 77 ? ? ?
tuple 3897 21737 299169 331236
weakref 37 ? ? ?
wrapper_descriptor 399 ? ? ?
Number collected: 9 Length of garbage: 0
Number of objects: 6739
Objects:
<class ActionFlowable> 1 ? ? 93
<class CMYKColor> 2 ? ? 418
<class Color> 143 ? ? 17859
<class Encoding> 1 ? ? 3089
<class Font> 2 ? ? 17592
<class Logger> 1 ? ? ?
<class MemoryError> 1 ? ? 52
<class NoEncryption> 2 ? ? 100
<class PCMYKColor> 2 ? ? 420
<class PDFStreamFilterBase85Encode> 1 ? ? 65
<class PDFStreamFilterZCompress> 1 ? ? 62
<class ParaParser> 1 ? ? ?
<class Sequencer> 1 ? ? 255
<class TableStyle> 5 ? ? 1977
<class TypeFace> 2 ? ? 11888
<class WarnOnce> 2 ? ? 202
<class _Environ> 1 36 ? 2282
<class _Feature> 3 ? ? 438
<class _FrameBreak> 1 ? ? 89
<class _Helper> 1 ? ? 25
<class _Name2StandardEncodingMap> 1 7 ? 14599
<class _Printer> 3 ? ? 1108
<class _ThreadSafeCounter> 1 ? ? ?
class 167 ? 1 6231
dict 595 14011 305269 388256
frame 2 ? ? ?
function 1053 ? ? 9223
getset_descriptor 18 ? ? ?
instance method 19 ? ? ?
list 93 1442 12838 17393
member_descriptor 91 ? ? ?
method_descriptor 112 ? ? ?
module 77 ? ? ?
tuple 3897 21737 299169 331236
weakref 37 ? ? ?
wrapper_descriptor 399 ? ? ?
--
Robin Becker