| Home | Trees | Indices | Help |
|
|---|
|
|
1 '''
2 Convert data blob into a Peach DOM based on a template specification.
3
4 The data cracker is still early in it's development lifecycle. Currently
5 it is capable of using well defined templates to crack data.
6
7 @author: Michael Eddington
8 @version: $Id: Peach.Engine.incoming-pysrc.html 1138 2008-08-16 19:39:03Z meddingt $
9 '''
10
11 #
12 # Copyright (c) 2007-2008 Michael Eddington
13 #
14 # Permission is hereby granted, free of charge, to any person obtaining a copy
15 # of this software and associated documentation files (the "Software"), to deal
16 # in the Software without restriction, including without limitation the rights
17 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
18 # copies of the Software, and to permit persons to whom the Software is
19 # furnished to do so, subject to the following conditions:
20 #
21 # The above copyright notice and this permission notice shall be included in
22 # all copies or substantial portions of the Software.
23 #
24 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
27 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
29 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 # SOFTWARE.
31 #
32
33 # Authors:
34 # Michael Eddington (mike@phed.org)
35
36 # $Id: Peach.Engine.incoming-pysrc.html 1138 2008-08-16 19:39:03Z meddingt $
37
38 import sys, os, time, struct, types, operator
39
40 sys.path.append("c:/peach")
41 sys.path.append(".")
42 sys.path.append("..")
43 sys.path.append("../..")
44
45 from Peach.Engine.common import *
46 #from Peach.Engine.parser import *
47 from Peach.Engine.dom import *
48 import Peach
49
51 '''
52 Debug output. Uncommenting the following
53 print line will cause *lots* of output
54 to be displayed. It significantly slows the
55 data cracking process.
56 '''
57 # Don't show look aheads
58 if Peach.Engine.engine.Engine.debug:
59 if DataCracker._tabLevel == 0:
60 print msg
61
63 '''
64 This class will try and parse data into a data model. This
65 process will try and best-fit data based on performing look
66 aheads with fit-ratings.
67 '''
68
69 _tabLevel = 0
70
72 self.peach = peachXml
73 self.deepString = -1
74 self.lookAheadDepth = 0
75 self.lookAhead = False
76 self.haveAllData = False
77
78 if not inner:
79 DataCracker._tabLevel = 0
80
82 '''
83 Crack data based on template. Set values into data tree.
84
85 Will throw an exception (NeedMoreData) if additional data is required.
86 The exception contains the minimum amount of additional data needed before
87 trying to re-crack the data.
88 '''
89
90 # Reset all values in tree
91 # NOTE: Do not change setValue to method. We NEEVER want
92 # to run this with setDefaultValue or else DEATH AND DOOM TO U!
93 self._resetDataElementValues(template, 'setValue')
94
95 #self.method = 'setValue'
96 self.method = method
97 (rating, pos) = self._handleNode(template, data, 0, None) #, self.dom)
98 Debug(1, "RATING: %d - POS: %d - LEN(DATA): %d" % (rating, pos, len(data)))
99
100 return (rating, pos)
101
103 '''
104 Reset values in data tree to None.
105 '''
106
107 eval("node.%s(None)" % method)
108
109 if hasattr(node, 'rating'):
110 delattr(node, 'rating')
111
112 if hasattr(node, 'pos'):
113 delattr(node, 'pos')
114
115 for child in node._children:
116 if isinstance(child, Peach.Engine.dom.DataElement):
117 self._resetDataElementValues(child, method)
118
119
121 '''
122 Determin the initial read size for this template.
123 '''
124
125 (rating, size) = self._handleBlockSize(template, None, None, None)
126 if rating < 4:
127 return size
128
129 return -1
130
132 '''
133 Try and determine size of a node.
134
135 Rating of:
136
137 1 - Exact
138 2 - Exact, but not everything
139 3 - Not complete to block yet
140 4 - Nadda
141
142 '''
143
144 if node == None:
145 raise Exception("Node is None, bailing!")
146
147 if node.elementType == 'string':
148 (rating, size) = self._handleStringSize(node, data, pos, parent)
149
150 elif node.elementType == 'number':
151 (rating, size) = self._handleNumberSize(node, data, pos, parent)
152
153 elif node.elementType == 'block' or node.elementType == 'template':
154 (rating, size) = self._handleBlockSize(node, data, pos, parent)
155
156 elif node.elementType == 'blob':
157 (rating, size) = self._handleBlobSize(node, data, pos, parent)
158
159 elif node.elementType == 'flags':
160 (rating, size) = self._handleFlagsSize(node, data, pos, parent)
161
162 elif node.elementType == 'choice':
163 rating = 1
164 size = 1
165
166 else:
167 raise Exception("Unknown elementType: %s" % node.elementType)
168
169 return (rating, size)
170
172
173 Debug(1, "_handleStringSize(%s)" % node.name)
174
175 ## TODO: Handle size relation!
176
177 if node.defaultValue != None and node.isStatic:
178 node.length = len(node.defaultValue)
179
180 if node.length != None:
181 return (1, int(node.length))
182
183 if node.defaultValue != None:
184 return (2, len(node.defaultValue))
185
186 return (2, 1)
187
189
190 Debug(1, "_handleBlobSize(%s)" % node.name)
191
192 ## TODO: Handle size relation!
193
194 if node.defaultValue != None and node.isStatic:
195 node.length = len(node.defaultValue)
196
197 if node.length != None:
198 return (1, int(node.length))
199
200 # This probably isn't a good idea, so lets not do it :)
201 #if node.defaultValue != None:
202 # return (2, len(node.defaultValue))
203
204 return (2, 1)
205
207 return (1, node.size / 8)
208
211
213
214 ## TODO: Handle maxOccurs!
215
216 Debug(1, "_handleBlockSize(%s)" % node.name)
217
218 curSize = 0
219 curRating = 0
220
221 for child in node._children:
222
223 if not isinstance(child, DataElement):
224 continue
225
226 (rating, size) = self._handleNodeSize(child, data, pos, node)
227 if rating < 3:
228 curSize += size
229 curRating = rating
230
231 elif rating == 3:
232 curSize += size
233 curRating = rating
234 break
235
236 else:
237 curRating = 3
238 break
239
240 return (curRating, curSize)
241
243 '''
244 Get the object indicated by ref. Currently the object must have
245 been defined prior to this point in the XML
246 '''
247
248 origStr = str
249 baseObj = self.peach
250
251 # Parse out a namespace
252
253 if str.find(":") > -1:
254 ns, tmp = str.split(':')
255 str = tmp
256
257 # Check for namespace
258 if hasattr(self.context.namespaces, ns):
259 baseObj = getattr(self.context.namespaces, ns)
260 else:
261 raise Exception("Unable to locate namespace")
262
263 for name in str.split('.'):
264
265 # check base obj
266 if hasattr(baseObj, name):
267 baseObj = getattr(baseObj, name)
268
269 # check templates
270 elif hasattr(baseObj, 'templates') and hasattr(baseObj.templates, name):
271 baseObj = getattr(baseObj.templates, name)
272
273 else:
274 raise Exception("Could not resolve ref", origStr)
275
276 return baseObj
277
279
280 root = node
281 while hasattr(root, 'parent') and root.parent != None:
282 root = root.parent
283
284 return root
285
287
288 # 1. Find root
289
290 root = node
291 while root.parent != None:
292 root = root.parent
293
294 # 2. Check if has a realParent
295
296 if hasattr(root, 'realParent'):
297 #print "_fixRealParent(): Found fake root: ", root.name
298 root.parent = root.realParent
299
300 # done!
301
303
304 # 1. Look for fake root
305
306 root = node
307 while not hasattr(root, 'realParent') and root.parent != None:
308 root = root.parent
309
310 # 2. Remove parent link
311 #print "_unFixRealParent(): Found fake root: ", root.name
312 root.parent = None
313
315
316 Debug(1, "_handleNode(%s): %s >>Enter" % (node.name, node.elementType))
317
318 # 0. Are we okay?
319
320 if pos > len(data):
321 raise Exception("Running past data!")
322
323 # 1. do we have a node?
324
325 if node == None:
326 raise Exception("Node is None, bailing!")
327
328 # 2. check for when relation
329
330 if node.HasWhenRelation():
331 rel = node.GetWhenRelation()
332
333 environment = {
334 #'Peach' : self.engine.peach,
335 'self' : node,
336 'pos' : pos,
337 'data' : data
338 }
339
340 Debug(1, "_handleNode: When: Running expression")
341 self._fixRealParent(node)
342
343 if not evalEvent(rel.when, environment):
344 # Remove this node from data tree
345 #print "Removing node: ", node.name
346
347 self._unFixRealParent(node)
348
349 #if DataCracker._tabLevel != 0 and node.name == 'ChunkIEND':
350 # #print "$$$$$$$$$ Removing chunkIEND!!!"
351 # print node.parent
352
353 node.parent.__delitem__(node.name)
354
355 Debug(1, "_handleNode: When: Returned False. Returning 1.")
356
357 return (1, pos)
358
359 self._unFixRealParent(node)
360 Debug(1, "_handleNode: When: Returned True.")
361
362 # Would be nice to have our current pos in scripting
363 node.possiblePos = pos
364
365 # 3. minOccurs == 0
366
367 ## TODO
368
369 # 3.5. Save origional copy
370
371 origionalNode = node.copy(node.parent)
372
373 #if node.parent != None:
374 # for c in node.parent:
375 # Debug(1, "---> parent.child.name: " + c.name)
376
377 # 4. do the crazy!
378
379 if node.elementType == 'string':
380 (rating, pos) = self._handleString(node, data, pos, parent, doingMinMax)
381
382 elif node.elementType == 'number':
383 (rating, pos) = self._handleNumber(node, data, pos, parent, doingMinMax)
384
385 elif node.elementType in ['block', 'template', 'choice']:
386
387 # First -- Determin if we know the size of this block via a
388 # size-of relation
389
390 length = None
391 relation = node.getRelationOfThisElement()
392 if relation != None and relation.type == 'size' and node.parent != None:
393 try:
394 length = relation.getValue(True)
395 Debug(1, "-----> FOUND BLOCK OF RELATION [%s] <-----" % repr(length))
396 fullName = relation.parent.getFullname()
397 Debug(1, "Size-of Fullname: " + fullName)
398 Debug(1, "node.parent Fullname: " + node.parent.getFullname())
399
400 #length = relation.getValue()
401 Debug(1, "Size-of Length: %s" % length)
402
403 except:
404 length = None
405
406 if length != None:
407 # Make sure we have the data
408 if len(data) < (pos+length):
409 if not self.haveAllData:
410 raise NeedMoreData(length, "")
411
412 else:
413 rating = 4
414 pos = pos + length
415
416 else:
417 Debug(1, "---- About to Crack internal Block ----")
418
419 # Parse this node on it's own
420 cracker = DataCracker(self.peach, True)
421 cracker.haveAllData = True
422 data = data[pos:pos+length]
423
424 # Do we have a transformer, if so decode the data
425 if node.transformer != None:
426 try:
427 data = node.transformer.transformer.decode(data)
428
429 except:
430 pass
431
432 # We need to remove the parent temporarily to
433 # avoid a recursion issue.
434 parent = node.parent
435 node.parent = None
436 node.realParent = parent
437
438 try:
439 (rating, crackpos) = cracker.crackData(node, data)
440 if rating == 0:
441 rating = 1
442
443 finally:
444 # We need to update the positions of each child
445 # to be + node.pos.
446 #
447 # Note: We are doing this in a finally to make
448 # sure the values in peach validation are
449 # correct.
450 for c in node.genAllChildDataElements():
451 if hasattr(c, 'pos') and c.pos != None:
452 c.pos += pos
453
454 node.parent = parent
455 node.realParent = None
456 node.pos = pos
457 node.rating = rating
458
459 pos += length
460
461 # Verify we used all the data
462 if crackpos != len(data):
463 Debug(1, "---- Crackpos != len(data): %d != %d ----" % (crackpos, len(data)))
464 rating = 4
465
466 Debug(1, "---- Finished with internal block (%d:%d) ----" % (rating, pos))
467
468 else:
469 if node.elementType == 'choice':
470 (rating, pos) = self._handleChoice(node, data, pos, parent, doingMinMax)
471 else:
472 (rating, pos) = self._handleBlock(node, data, pos, parent, doingMinMax)
473
474 elif node.elementType == 'blob':
475 (rating, pos) = self._handleBlob(node, data, pos, parent, doingMinMax)
476 Debug(1, "---] pos = %d" % pos)
477 elif node.elementType == 'flags':
478 (rating, pos) = self._handleFlags(node, data, pos, parent, doingMinMax)
479
480 else:
481 raise str("Unknown elementType: %s" % node.elementType)
482
483 # 5. Handle maxOccurs > 1
484
485 if rating < 3 and node.maxOccurs > 1 and not doingMinMax:
486 Debug(1, "*** NOde Occures more then once!")
487 occurs = 1
488 newRating = rating
489 newCurPos = pos
490 dom = None
491 curpos = None
492 maxOccurs = node.maxOccurs
493 name = node.name
494 goodLookAhead = None
495
496 try:
497
498 # Locate any count restrictions and update maxCount to match
499
500 relation = node.getRelationOfThisElement()
501 if relation != None and relation.type == 'count' and node.parent != None:
502 try:
503 maxOccurs = int(relation.getValue(True))
504 Debug(1, "@@@ Found count relation [%d]" % maxOccurs)
505
506 except:
507 pass
508
509
510 Debug(1, "@@@ Entering while loop")
511 while occurs < maxOccurs and rating < 3:
512 Debug(1, "@@@ In While, newCurPos=%d" % newCurPos)
513
514 # 0. Are we out at end of stream?
515 if self.haveAllData and newCurPos >= len(data):
516 Debug(1, "@ Exiting while loop, end of data! YAY!")
517 break
518 else:
519 Debug(1, "@ Have enough data to try again: %d < %d" % (newCurPos, len(data)))
520
521 # 1. Make a copy so we don't overwrite
522 # existing node
523 nodeCopy = origionalNode.copy(node.parent)
524 nodeCopy.name = name + "-%d" % occurs
525 node.parent.insert(node.parent.index(node)+occurs, nodeCopy)
526
527 # 1.1. Run onArrayNext
528
529 if DataCracker._tabLevel == 0 and nodeCopy.onArrayNext != None:
530 evalEvent( node.onArrayNext, { 'node' : nodeCopy } )
531
532 # 2. Check out look-ahead
533 if self._nextNode(nodeCopy) != None:
534 Debug(1, "*** >> node.name: %s" % node.name)
535 Debug(1, "*** >> nodeCopy.name: %s" % nodeCopy.name)
536 Debug(1, "*** >> LookAhead")
537
538 newRating = self._lookAhead(nodeCopy, data, newCurPos, None, False)
539 Debug(1, "*** << LookAhead [%d]" % newRating)
540
541 # If look ahead was good, save and check later.
542 if newRating < 3:
543 goodLookAhead = newRating
544 #node.parent.__delitem__(nodeCopy.name)
545 #Debug(1, "*** Exiting min/max: Look ahead was too good!")
546 #break
547
548 # 3. Do actual
549 Debug(1, "*** >> nodeCopy.name: %s" % nodeCopy.name)
550 Debug(1, "*** >> DOING ACTUAL HANDLENODE")
551
552 (newRating, newCurPos) = self._handleNode(nodeCopy, data, newCurPos, None, True)
553
554 # Verify we didn't break a good lookahead
555 if newRating < 3 and not self.lookAhead and goodLookAhead != None:
556 lookAheadRating = self._lookAhead(nodeCopy, data, newCurPos, None, False)
557 if lookAheadRating >= 3:
558 del node.parent[nodeCopy.name]
559 Debug(1, "*** Exiting min/max: We broke a good lookAhead!")
560 break
561
562 # 4. Verify high enough rating
563 if newRating < 3 and not self.lookAhead:
564 Debug(1, "*** That worked out!")
565 pos = curpos = newCurPos
566 rating = newRating
567
568 # First time through convert position 0 node
569 if occurs == 1:
570 # First fix up our first node
571 index = node.parent.index(node)
572 del node.parent[node.name]
573
574 node.array = node.name
575 node.name = node.name + "-0"
576 node.arrayPosition = 0
577 node.arrayMinOccurs = node.minOccurs
578 node.arrayMaxOccurs = node.maxOccurs
579 node.minOccurs = 1
580 node.maxOccurs = 1
581
582 node.parent.insert(index, node)
583
584 # Next fix up our copied node
585 nodeCopy.array = node.array
586 nodeCopy.arrayPosition = occurs
587 nodeCopy.arrayMinOccurs = node.arrayMinOccurs
588 nodeCopy.arrayMaxOccurs = node.arrayMaxOccurs
589 nodeCopy.minOccurs = 1
590 nodeCopy.maxOccurs = 1
591
592 # Insert into parent at correct position
593 #node.parent.insert(node.parent.index(node)+occurs, nodeCopy)
594
595 else:
596 Debug(1, "*** Didn't work out!")
597 node.parent.__delitem__(nodeCopy.name)
598 break
599
600 occurs += 1
601
602 Debug(1, "@@@ Looping, occurs=%d, rating=%d" % (occurs, rating))
603
604 Debug(1, "@@@ Exiting While Loop")
605
606 except:
607 #pass
608 raise
609
610 if curpos != None:
611 Debug(1, "@@@ Returning a curpos=%d, pos=%d, newCurPos=%d, occuurs=%d" %(curpos, pos, newCurPos, occurs))
612 return (rating, curpos)
613
614 Debug(1, "_handleNode(%s): type=%s, pos=%d, rating=%d <<EXIT" % (node.name, node.elementType, pos, rating))
615 return (rating, pos)
616
618 '''
619 Look ahead one step and get the next rating. Looking ahead
620 from a current node is more complex than it might first seem.
621
622 For example, we might be the 2nd to last element in a block
623 that is part of a larger block (but not the last element). We
624 will need to be tricky inorder to properly look ahead at the
625 rest of the document.
626 '''
627
628 if node == None:
629 return 1
630
631 if pos > len(data):
632 Debug(1, "_lookAhead(): pos > len(data), no lookahead")
633 return 4
634 #else:
635 # Debug(1, "_lookAhead(): pos[%d] < len(data)[%d]" % (pos, len(data)))
636
637 # Setup a few variables
638
639 DataCracker._tabLevel += 1
640
641 self.lookAhead = True
642 self.lookAheadDepth += 1
643
644 origNode = node
645 origParent = parent
646
647 # First lets copy the data model
648
649 root = origNode.getRootOfDataMap().copy(None)
650 root._FixParents()
651
652 node = root.findDataElementByName(origNode.getFullDataName())
653 sibling = self._nextNode(node)
654
655 if node == None:
656 raise Exception("Node should not be null here! [%s]" % origNode.getFullDataName())
657
658 if origParent != None:
659 parent = root.findDataElementByName(origParent.getFullDataName())
660
661 #if DataCracker._tabLevel == 1:
662 # print "_lookAhead: node.name = %s" % node.name
663 # print "_lookAhead: sibling.name = %s" % sibling.name
664
665 # If we could have more than one of the curret node
666 # we will try that node again UNLESS we minMax == False
667 if node.maxOccurs > 1 and minMax:
668 Debug(1, "_lookAhead(): look ahead for node")
669
670 try:
671 (rating, pos) = self._handleNode(node, data, pos, parent)
672
673 # If we have a good rating return it
674 if rating < 3:
675
676 self.lookAheadDepth -= 1
677 if self.lookAheadDepth == 0:
678 self.lookAhead = False
679
680 self.lookAhead = False
681 DataCracker._tabLevel -= 1
682 return rating
683
684 except NeedMoreData:
685 self.lookAheadDepth -= 1
686 if self.lookAheadDepth == 0:
687 self.lookAhead = False
688
689 DataCracker._tabLevel -= 1
690 return 4
691
692 # Now lets try that sibling if we can
693
694 if sibling == None:
695
696 # if no sibling than everything is okay
697
698 Debug(1, "_lookAhead(): node.nextSibling() == None, returning 1")
699 #print "_lookAhead(): node.nextSibling() == None, returning 1"
700 rating = 1
701
702 else:
703
704 try:
705 Debug(1, "_lookAhead(): look ahead for node.Sibling(): %s->%s" % (node.name,sibling.name))
706 #print "_lookAhead(): look ahead for node.Sibling(): %s->%s" % (node.name,sibling.name)
707 (rating, pos) = self._handleNode(sibling, data, pos, parent)
708
709 except NeedMoreData:
710 ## Should we hide this??
711 #raise e
712 self.lookAheadDepth -= 1
713 if self.lookAheadDepth == 0:
714 self.lookAhead = False
715
716 DataCracker._tabLevel -= 1
717 return 4
718
719 self.lookAheadDepth -= 1
720 if self.lookAheadDepth == 0:
721 self.lookAhead = False
722
723 DataCracker._tabLevel -= 1
724 if pos < len(data):
725 return rating + 1
726
727 else:
728 return rating
729
730
732 '''
733 Locate the next node.
734
735 1. Do we have a .nextSibling?
736 2. Does are parent have .nextSibling?
737 ...
738
739 Need to also support escaping Choice blocks!
740 '''
741
742 if node == None:
743 return None
744
745 try:
746 Debug(1, "_nextNode(%s)" % node.name)
747
748 except:
749 Debug(1, "_nextNode: %s" % repr(node))
750 raise
751
752 if not isinstance(node, Peach.Engine.dom.DataElement) or \
753 node.elementType == 'template':
754
755 #Debug(1, "_nextNode(%s): not data element or is template")
756
757 return None
758
759 # Try and escape Choice blocks.
760 if node.parent != None and node.parent.elementType == 'choice':
761 node = node.parent
762
763 nextNode = node.nextSibling()
764 while nextNode != None and not isinstance(nextNode, Peach.Engine.dom.DataElement):
765 nextNode = nextNode.nextSibling()
766
767 if nextNode != None and isinstance(nextNode, Peach.Engine.dom.DataElement):
768 return nextNode
769
770 return self._nextNode(node.parent)
771
773 if lookAheadRating == 2 and rating == 1:
774 rating = 2
775 elif rating < 3 and lookAheadRating > 2:
776 return rating - 1
777 elif rating == 3 and lookAheadRating > 3:
778 return rating - 1
779
780 return rating
781
783 Debug(1, "---> %s (%d)" % (node.name,pos))
784
785 # Default is failure
786 rating = 4
787 curpos = pos
788 node.currentElement = None
789
790 # Our list can shrink/expand as we go
791 # so lets copy the list up front.
792 children = []
793 for child in node._children:
794 if isinstance(child, DataElement):
795 children.append(child)
796
797 # Look for first child that matches, forget the rest.
798 for child in children:
799
800 # Skip any children created during array expantion
801 # they should already have values 'n all that good
802 # stuff :)
803 if hasattr(child, 'array') and child.array != None:
804 continue
805
806 # Try this child
807
808 Debug(1, "_handleChoice(): Tring child [%s]" % child.name)
809
810 (childRating, newpos) = self._handleNode(child, data, curpos)
811 if child.currentValue != None and len(child.currentValue) > 30:
812 Debug(1, "_handleChoice(): Rating: (%d) [%s]: %s = [%s]" % (childRating, repr(child.defaultValue), child.name, child.currentValue[:30]))
813 else:
814 Debug(1, "_handleChoice(): Rating: (%d) [%s]: %s = [%s]" % (childRating, repr(child.defaultValue), child.name, child.currentValue))
815
816 # Check if we are keeping this child or not
817 if childRating > 2:
818 Debug(1, "_handleChoice(): Child did not meet requirements, NEXT!")
819 continue
820
821 # Keep this child
822 Debug(1, "_handleChoice(): Keeping child [%s]" % child.name)
823 node.currentElement = child
824 rating = childRating
825 curpos = newpos
826
827 # TODO: Lets not remove the kids, but for now to keep things
828 # simple, we will look like a block after this so to
829 # speek.
830 for c in children:
831 if c != node.currentElement:
832 Debug(1, "_handleChoice(): Removing unused child [%s]" % c.name)
833 node.__delitem__(c.name)
834
835 break
836
837 Debug(1, "Choice rating: %d" % rating)
838 Debug(1, "<--- %s (%d through %d)" % (node.name,pos,newpos))
839
840 node.pos = pos
841 node.rating = rating
842 return (rating, curpos)
843
845
846 # Not going to handle alignment right now :)
847
848 Debug(1, "---> %s (%d)" % (node.name,pos))
849
850 rating = 0
851 ratingCnt = 0
852 ratingTotal = 0
853 curpos = pos
854
855 # Our list can shrink/expand as we go
856 # so lets copy the list up front.
857 children = []
858 for child in node._children:
859 children.append(child)
860
861 for child in children:
862
863 if not isinstance(child, DataElement):
864 continue
865
866 # Skip any children created during array expantion
867 # they should already have values 'n all that good
868 # stuff :)
869 if hasattr(child, 'array') and child.array != None:
870 continue
871
872 # Should we skip this child?
873 if child.minOccurs == 0 and self._nextNode(child) != None:
874 Debug(1, "_handleBlock(%s) >> minOccurs LookAhead [%s]" % (node.name,node.name))
875 newRating = self._lookAhead(self._nextNode(child), data, curpos, None, False)
876
877 Debug(1, "_handleBlock() << minOccurs LookAhead [%d]" % newRating)
878 if newRating < 3:
879 Debug(1, "_handleBlock(%s) !! Exiting minOccurs due to good lookAhead!" % node.name)
880 continue
881
882 else:
883 Debug(1, "_handleBlock(%s) !! Not existing minOccurs due to lookAhead" % node.name)
884
885 # Do the needfull
886
887 ratingCnt += 1
888
889 (childRating, newpos) = self._handleNode(child, data, curpos)
890 if child != None and child.currentValue != None and len(child.currentValue) > 30:
891 Debug(1, "_handleBlock(%s): Rating: (%d) [%s]: %s = [%s]" % (node.name, childRating, repr(child.defaultValue), child.name, child.currentValue[:30]))
892 else:
893 Debug(1, "_handleBlock(%s): Rating: (%d) [%s]: %s = [%s]" % (node.name, childRating, repr(child.defaultValue), child.name, repr(child.currentValue)))
894
895 if childRating > 2:
896 Debug(1, "_handleBlock(%s): Child rating sucks, exiting" % node.name)
897 rating = childRating
898 break
899
900 ratingTotal += childRating
901 if childRating > rating:
902 rating = childRating
903
904 curpos = newpos
905
906 if childRating >= 3:
907 newRating = self._lookAhead(child, data, curpos, None, False)
908 if newRating < 3:
909 childRating = 2
910 Debug(1, "_handleBlock(%s): Passing child due to lookAhead rating..." % node.name)
911 break
912 else:
913 Debug(1, "_handleBlock(%s): Child didn't pass, lookAhead not good!" % node.name)
914
915 (childRating, amount) = self._handleNodeSize(child, data, curpos, None)
916 amount = int(amount)
917 if amount == None or amount < 1:
918 size = 1
919
920 if not self.lookAhead and (amount+curpos) > len(data):
921 Debug(1, "*---* Block [%s] is raising needmoredata (%d bytes) for %s" % (node.name,(amount+curpos)-len(data), child.name))
922 raise NeedMoreData(amount, child.name)
923
924 Debug(1, "BLOCK RATING: %d" % rating)
925 Debug(1, "<--- %s (%d)" % (node.name,pos))
926 node.pos = pos
927 node.rating = rating
928 return (rating, curpos)
929
931 '''
932 Take a fullname (blah.blah.blah) and locate
933 it in our data dom.
934 '''
935 dom = self._getRootParent(dom)
936 obj = dom
937
938 for part in name.split('.'):
939 Debug(2, "_getDataFromFullname(%s): [%s]" % (name, obj.name))
940 if part == obj.name:
941 continue
942
943 obj = obj[part]
944
945 return obj[obj.name]
946
948 '''
949 Returns the rating and string. The rating is
950 how well we matched.
951
952 Rating:
953
954 1 - BEST If our default matched and look ahead is 1
955 2 - GOOD If our default matched and look ahead is 2
956 3 - OK If our look ahead is 1 or 2
957 4 - MPH If look ahead is 3 or 4
958 '''
959
960 # We just break from this to return values
961 while True:
962
963 Debug(1, "---> %s (%d)" % (node.name,pos))
964
965 self.deepString += 1
966
967 rating = 0
968 newpos = 0
969 length = None
970
971 # If we are static we should know our
972 # length.
973 if node.length == None and node.isStatic:
974 try:
975 node.length = len(node.defaultValue)
976 except:
977 raise PeachException("Error: String %s doens't have a default value, yet is marked isStatic." % node.name)
978
979 # Determin if we have a size-of relation
980 # and set our length accordingly.
981 relation = node.getRelationOfThisElement()
982 if relation != None and relation.type == 'size':
983 # we have a size-of relation
984 Debug(1, "** FOUND SIZE OF RELATION ***")
985
986 fullName = relation.parent.getFullname()
987 Debug(1, "Size-of Fullname: " + fullName)
988
989 length = relation.getValue(True)
990 Debug(1, "Size-of Length: %s" % length)
991
992 # Value may not be available yet
993 try:
994 length = int(length)
995 except:
996 pass
997
998 # Do we know our length?
999 if node.getLength() != None or length != None:
1000
1001 if length == None:
1002 length = node.getLength()
1003
1004 Debug(1, "_handleString: Found length of: %d" % length)
1005
1006 if node.type == 'wchar':
1007 length = length * 2
1008
1009 if len(data) < (pos + length):
1010 if not self.haveAllData:
1011 raise NeedMoreData((pos + length) - len(data), "")
1012
1013 else:
1014 rating = 4
1015 value = ""
1016 newpos = pos + length
1017 Debug(1, "_handleString: Want %d, have %d" % ((pos+length), len(data)))
1018 break
1019
1020 else:
1021
1022 value = data[pos:pos+length]
1023 newpos = pos + length
1024 defaultValue = node.defaultValue
1025
1026 if node.type == 'wchar':
1027 # convert to ascii string
1028 defaultValue = node.defaultValue.decode("utf-16le")
1029
1030 rating = 2
1031 if node.isStatic:
1032 if value != node.defaultValue and node.isStatic:
1033 Debug(1, "%s_handleString: %s: Bad match, static, but default didn't match [%s != %s]" % ('\t'*self.deepString, node.name, repr(value), repr(node.defaultValue)))
1034 rating = 4
1035
1036 else:
1037 Debug(1, "%s_handleString: %s: By length [%s]" % ('\t'*self.deepString, node.name, repr(value)))
1038 rating = 1
1039
1040 break
1041
1042 # Are we null terminated?
1043 elif node.nullTerminated:
1044 value = ''
1045
1046 newpos = pos
1047 rating = 666
1048
1049 if node.type != 'wchar':
1050 newpos = data.find('\0', pos)
1051
1052 if newpos == -1:
1053 if self.haveAllData:
1054 rating = 4
1055 value = ''
1056
1057 else:
1058 raise NeedMoreData(1, "Looking for string null terminator")
1059
1060 newpos += 1 # find leaves us a position down, need to add one to get the null
1061 value = data[pos:newpos]
1062 rating = 2
1063 break
1064
1065 elif node.type == 'wchar':
1066
1067 for newpos in xrange(pos, len(data), 2):
1068 if data[newpos] != '\0' and data[newpos+1] != '\0':
1069 value = data[pos:newpos]
1070 rating = 2
1071 break
1072
1073 if rating > 2:
1074 if self.haveAllData:
1075 rating = 4
1076 value = ''
1077
1078 else:
1079 raise NeedMoreData(1, "")
1080
1081 break
1082
1083 elif node.isStatic:
1084
1085 # first, look for our defaultValue
1086 if node.defaultValue == None:
1087 raise PeachException("Error: %s is marked as static but has no default value." % node.name)
1088
1089 Debug(1, "%s_handleString: %s: Found default value, doing checks" % ('\t'*self.deepString, node.name))
1090
1091 if node.type == 'wchar':
1092 defaultValue = str(node.defaultValue.decode("utf-16le"))
1093
1094 else:
1095 defaultValue = node.defaultValue
1096
1097 newpos = pos+len(defaultValue)
1098 value = data[pos:newpos]
1099 if value == defaultValue:
1100 rating = 2
1101 break
1102
1103 else:
1104 rating = 4
1105 Debug(1, "%s_handleString: %s: No match [%s == %s] @ %d" % ('\t'*self.deepString, node.name, repr(data[newpos:newpos+len(defaultValue)]), repr(defaultValue), pos))
1106 break
1107
1108 else:
1109
1110 # If we don't have a length, we try for a best fit
1111 # by adjusting the position until our look ahead has a rating
1112 # of 1 or 2.
1113
1114 # Are we the last data element?
1115 if self._nextNode(node) == None:
1116 if not self.haveAllData:
1117 self.haveAllData = True
1118 raise NeedMoreData(-1, "")
1119
1120 else:
1121 # Keep all the data :)
1122 Debug(1, "_handleString: Have all data, keeping it for me :)")
1123 value = data[pos:]
1124 newpos = len(data)
1125 rating = 1
1126 #break
1127
1128 else:
1129
1130 # Will will suckup bytes one by one check the
1131 # look ahead each time to see if we should keep
1132 # sucking.
1133 #
1134 # Note: Turns out running the lookAhead each time is slow.
1135
1136 lookRating = 666
1137 newpos = pos
1138 dataLen = len(data)
1139
1140 # If we have a following static just scan
1141 # for it instead of calling lookAhead.
1142 nextNode = self._nextNode(node)
1143 if nextNode.isStatic:
1144 nextValue = nextNode.getValue()
1145 nextValueLen = len(nextValue)
1146
1147 newpos = data.find(nextValue, pos)
1148 if newpos == -1:
1149 if self.haveAllData:
1150 value = ""
1151 rating = 4
1152 break
1153 else:
1154 raise NeedMoreData(1, "Couldn't locate nextValue [%s] in [%s]" % (repr(nextValue), repr(data[pos:])))
1155
1156 value = data[pos:newpos]
1157 rating = 2
1158 break
1159
1160 # This loop is slow!
1161 while lookRating > 2 and newpos < dataLen:
1162 newpos += 1
1163 lookRating = self._lookAhead(node, data, newpos, parent)
1164
1165 value = data[pos:newpos]
1166
1167 if lookRating > 2:
1168 rating = 3
1169
1170 else:
1171 rating = 2
1172
1173 break
1174
1175 break
1176
1177 # Deal with wchar
1178 if node.type == 'wchar':
1179 value = str(value.decode("utf-16le"))
1180
1181 # Set value
1182 eval("node.%s(value)" % self.method)
1183
1184 # Are we last?
1185 if self._nextNode(node) == None:
1186
1187 # Note: If doingMinMax then we can't
1188 # assume we should eat all data even
1189 # if we are the last node!
1190 #
1191 # Note2: maxOccurs can lie if we are doingMinMax!
1192 #
1193 if newpos < len(data) and node.maxOccurs == 1 and (node.parent == None or node.parent.maxOccurs == 1) and not doingMinMax:
1194 # We didn't use it all up, sad for us!
1195 Debug(1, "--- Didn't use all data, rating == 4")
1196 rating = 4
1197
1198 # Return values
1199
1200 Debug(1, "<--- %s (%d, %d-%d)" % (node.name, rating, pos, newpos))
1201
1202 node.pos = pos
1203 node.rating = rating
1204 self.deepString -= 1
1205 return (rating, newpos)
1206
1208 '''
1209 Handle Number. Return (rating, newpos, value) in tuple.
1210
1211 Rating:
1212
1213 1 - BEST If our default matched and look ahead is 1
1214 2 - GOOD If our default matched and look ahead is 2
1215 3 - OK If our look ahead is 1 or 2
1216 4 - MPH If look ahead is 3 or 4
1217
1218 '''
1219
1220 Debug(1, "---> %s (%d)" % (node.name,pos))
1221
1222 node.rating = 0
1223 length = node.size/8
1224
1225 # See if we have enough data
1226
1227 if (pos+length) > len(data):
1228 # need more
1229 raise NeedMoreData((pos+length) - len(data), node.name)
1230
1231 # Get value based on element length
1232
1233 value = data[pos:pos+length]
1234 newpos = pos + length
1235
1236 # Build format string
1237
1238 fmt = ''
1239
1240 if node.endian == 'little':
1241 fmt = '<'
1242 else:
1243 fmt = '>'
1244
1245 if node.size == 8:
1246 fmt += 'b'
1247 elif node.size == 16:
1248 fmt += 'h'
1249 elif node.size == 32:
1250 fmt += 'i'
1251 elif node.size == 64:
1252 fmt += 'q'
1253
1254 if not node.signed:
1255 fmt = fmt.upper()
1256
1257 # Unpack value
1258
1259 value = str(struct.unpack(fmt, value)[0])
1260
1261 # Adjust rating based on defaultValue
1262
1263 if node.isStatic:
1264 if value != str(node.defaultValue):
1265 Debug(1, "_handleNumber: Number is static but did not match, failing. [%s] != [%s]" % (value, node.defaultValue))
1266 node.rating = 4
1267 else:
1268 Debug(1, "_handleNumber: Number is static and matched. [%s] == [%s]" % (value, node.defaultValue))
1269 node.rating = 1
1270 else:
1271 node.rating = 2
1272
1273 # Set value on data element
1274
1275 eval("node.%s(value)" % self.method)
1276
1277 # Return all of it
1278 node.pos = pos
1279
1280 Debug(1, "<--- %s (%d, %d-%d)" % (node.name, node.rating, pos, newpos))
1281 return (node.rating, newpos)
1282
1284 '''
1285 Returns the rating and string. The rating is
1286 how well we matched.
1287
1288 Rating:
1289
1290 1 - BEST If our default matched and look ahead is 1
1291 2 - GOOD If our default matched and look ahead is 2
1292 3 - OK If our look ahead is 1 or 2
1293 4 - MPH If look ahead is 3 or 4
1294 '''
1295
1296 Debug(1, "---> %s (%d)" % (node.name,pos))
1297
1298 rating = 0
1299 length = node.length/8
1300
1301 if (pos+length) > len(data):
1302 # need more
1303 #print "_handleFlags(): We need %d we have total %d and are at pos %d" % (length, len(data), pos)
1304 raise NeedMoreData((pos+length) - len(data), node.name)
1305
1306 value = data[pos:pos+length]
1307 newpos = pos + length
1308
1309 # Now, unpack the integer
1310
1311 fmt = ''
1312
1313 if node.endian == 'little':
1314 fmt = '<'
1315 else:
1316 fmt = '>'
1317
1318 if node.length == 8:
1319 fmt += 'B'
1320 elif node.length == 16:
1321 fmt += 'H'
1322 elif node.length == 32:
1323 fmt += 'I'
1324 elif node.length == 64:
1325 fmt += 'Q'
1326
1327 value = int(struct.unpack(fmt, value)[0])
1328 #print "_handleFlags(): Value: %d" % value
1329
1330 # Now, do the Flag portions
1331
1332 for child in node._children:
1333 if child.elementType != 'flag':
1334 continue
1335
1336 childValue = value >> child.position
1337
1338 mask = 0
1339 for i in range(0, child.length):
1340 mask += 1 << i
1341
1342 #print "_handleFlags(): %d, %d, %d" % (childValue, mask, childValue & mask)
1343 childValue = childValue & mask
1344
1345 # Set child node value
1346 eval("child.%s(int(childValue))" % self.method)
1347
1348 # Determin rating
1349
1350 rating = 2
1351
1352 Debug(1, "<--- %s (%d, %d-%d)" % (node.name, rating, pos, newpos))
1353
1354 node.pos = pos
1355 node.rating = rating
1356 return (rating, newpos)
1357
1359 '''
1360 Returns the rating and string. The rating is
1361 how well we matched.
1362
1363 Rating:
1364
1365 1 - BEST If our default matched and look ahead is 1
1366 2 - GOOD If our default matched and look ahead is 2
1367 3 - OK If our look ahead is 1 or 2
1368 4 - MPH If look ahead is 3 or 4
1369 '''
1370
1371 Debug(1, "---> %s (%d)" % (node.name,pos))
1372
1373 rating = 0
1374 newpos = 0
1375 length = None
1376 hasSizeofRelation = False
1377
1378 # Determin if we have a size-of relation
1379 # and set our length accordingly.
1380 relation = node.getRelationOfThisElement()
1381 if relation != None and relation.type == 'size':
1382 # we have a size-of relation
1383 Debug(1, "** FOUND SIZE OF RELATION ***")
1384
1385 fullName = relation.parent.getFullname()
1386 Debug(1, "Size-of Fullname: " + fullName)
1387
1388 length = relation.getValue(True)
1389 Debug(1, "Size-of Length: %s" % length)
1390
1391 # We might not be ready to get this
1392 # value yet (look head), but try
1393 try:
1394 length = int(length)
1395 except:
1396 pass
1397 else:
1398 Debug(1, "_handleBlob: No relation found")
1399
1400 # Do we know our length?
1401 if node.getLength() != None or length != None:
1402
1403 if length == None:
1404 length = node.getLength()
1405
1406 value = data[pos:pos + length]
1407 newpos = pos + length
1408 rating = 2
1409
1410 if value == node.defaultValue:
1411 rating = 1
1412 elif node.isStatic:
1413 rating = 4
1414
1415 if (pos+length) > len(data):
1416 if not self.haveAllData:
1417 raise NeedMoreData((pos+length) - len(data), "")
1418 else:
1419 Debug(1, "_handleBlob: Not enough data, rating = 4: %d left" % (len(data)-pos))
1420 rating = 4
1421
1422 else:
1423 # If we don't have a sizeof relation, we try for a best fit
1424 # by adjusting the position until our look ahead has a rating
1425 # of 1 or 2.
1426
1427 # Are we the last data element?
1428 if self._nextNode(node) == None:
1429 #print "--- Last element, snafing it all :)"
1430
1431 if not self.haveAllData:
1432 self.haveAllData = True
1433 raise NeedMoreData(-1, "")
1434
1435 else:
1436 # Keep all the data :)
1437 value = data[pos:]
1438 newpos = len(data)
1439 rating = 1
1440
1441 else:
1442 #if self.haveAllData:
1443 # print "--- Was not last node"
1444
1445 lookRating = 666
1446 newpos = pos
1447
1448 # If we have a following static just scan
1449 # for it instead of calling lookAhead.
1450 nextNode = self._nextNode(node)
1451 if nextNode.isStatic:
1452 nextValue = nextNode.getValue()
1453 nextValueLen = len(nextValue)
1454
1455 newpos = data.find(nextValue, pos)
1456 if newpos == -1:
1457 raise NeedMoreData(1, "Couldn't locate nextValue(%s) [%s]" % (nextNode.getFullnameInDataModel(), nextValue))
1458
1459 value = data[pos:newpos]
1460 rating = 2
1461
1462 else:
1463
1464 while lookRating > 2 and newpos < len(data):
1465 #Debug(1, ".")
1466 newpos += 1
1467 lookRating = self._lookAhead(node, data, newpos, parent)
1468 #Debug(1, "newpos: %d lookRating: %d data: %d" % (newpos, lookRating, len(data)))
1469
1470 while lookRating <= 2 and newpos < len(data):
1471 #Debug(1, ",")
1472 newpos += 1
1473 lookRating = self._lookAhead(node, data, newpos, parent)
1474 #Debug(1, "newpos: %d lookRating: %d data: %d" % (newpos, lookRating, len(data)))
1475
1476 #if newpos >= len(data):
1477 # newpos -= 1
1478 # #raise str("Unable to parse out blob %s" % node.name)
1479
1480 value = data[pos:newpos]
1481 rating = 2
1482
1483 #print "Found blob: [%s]" % value
1484
1485 eval("node.%s(value)" % self.method)
1486
1487 Debug(1, "<--- %s (%d, %d-%d)" % (node.name, rating, pos, newpos))
1488
1489 node.pos = pos
1490 node.rating = rating
1491 return (rating, newpos)
1492
1503
1518
1519 # end
1520
| Home | Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Sat Aug 16 12:17:18 2008 | http://epydoc.sourceforge.net |