made pbkdf2 funuctional in python 3

started refactoring other code to work in python 3

--HG--
branch : mung

.hgignore

@@ -1,2 +1,5 @@
 syntax: glob
 *.pyc
+bin
+lib
+include

bi.py

@@ -1,4 +1,4 @@
-from builtins_code import bi
+from bi_code import bi
 
 __all__ = ['builtin_map']
 

bytecode.py

@@ -1,5 +1,5 @@
 from language_tools import *
-from builtins import builtin_map
+from bi import builtin_map
 import optimizer
 
 
@@ -11,7 +11,7 @@ def coerce(value):
 		return VMInteger(value)
 	elif isinstance(value, (tuple, list)):
 		return VMList(list(value))
-	elif isinstance(value, unicode):
+	elif isinstance(value, str):
 		return VMString(value)
 	elif isinstance(value, dict):
 		return VMTable(value)
@@ -298,7 +298,7 @@ class UnaryOp(CodeOp):
 		rv = []
 		ops = []
 		for t in tokens:
-			if isinstance(t, (str, unicode)) and t in UnaryOp.map:
+			if isinstance(t, (str, bytes)) and t in UnaryOp.map:
 				ops.append(UnaryOp(UnaryOp.map[t]))
 			else:
 				rv.append(t)
@@ -524,7 +524,7 @@ class WhileBlock(CodeOp):
 	def parse(tokens):
 		rv = WhileBlock()
 		
-		for i in xrange(1, len(tokens)):
+		for i in range(1, len(tokens)):
 			tok = tokens[i]
 			if rv.cond == None:
 				rv.cond = tok
@@ -583,7 +583,7 @@ class TryBlock(CodeOp):
 		rv = TryBlock()
 		
 		active_tok = None
-		for i in xrange(1, len(tokens)):
+		for i in range(1, len(tokens)):
 			tok = tokens[i]
 			if tok in ("try", "except", "else", "finally"):
 				active_tok = tok
@@ -677,7 +677,7 @@ class ForeachBlock(CodeOp):
 	def parse(tokens):
 		rv = ForeachBlock()
 		
-		for i in xrange(1, len(tokens)):
+		for i in range(1, len(tokens)):
 			tok = tokens[i]
 			if rv.var == None:
 				rv.var = tok
@@ -717,7 +717,7 @@ class IfBlock(CodeOp):
 		conds = []
 		tok_count = 0
 		active_tok = None
-		for i in xrange(len(tokens)):
+		for i in range(len(tokens)):
 			tok = tokens[i]
 
 			if tok == "endif":

language_types.py

@@ -12,7 +12,7 @@ def disallow_keywords(tokens,keywords=None):
 		if isinstance(t, VMIdent):
 			if t.name in keywords:
 				raise ParseException("Restricted keyword: %s" % (t.name,))
-		elif isinstance(t, unicode):
+		elif isinstance(t, str):
 			tstr = t.encode('ascii', 'ignore')
 			if tstr in keywords:
 				raise ParseException("Restricted keyword: %s" % (tstr,))
@@ -98,10 +98,10 @@ class VMTablePair(VMType):
 class VMString(VMType):
 	def __init__(self, value):
 		VMType.__init__(self)
-		if isinstance(value, unicode):
+		if isinstance(value, str):
 			self.value = value
 		else:
-			self.value = unicode(str(value), 'ascii', 'ignore')
+			self.value = str(value, 'ascii', 'ignore')
 	
 	def __repr__(self):
 		return "\"%s\"" % (repr(self.value)[1:].strip("'").replace("\\'", "'").replace('"', '\\"'),)
@@ -138,7 +138,7 @@ class VMIdent(VMRef):
 		self.name = name
 		
 	def bytecode(self):
-		return [StackLiteral(unicode(self.name))]
+		return [StackLiteral(str(self.name))]
 
 	@staticmethod
 	@tokenparser
@@ -156,7 +156,7 @@ class VMVariable(VMRef):
 		self.name = name
 
 	def ref(self):
-		return [StackLiteral(unicode(self.name))]
+		return [StackLiteral(str(self.name))]
 
 	def bytecode(self):
 		return codejoin(self.ref(), GetVariable())

listener.py

@@ -92,12 +92,12 @@ class Connection(object):
 		data = self.conn.recv(bytes)
 
 		if self.input_encoding == 'raw':
-			return unicode(self.escape(data), 'ascii')
+			return str(self.escape(data), 'ascii')
 		else:
-			return unicode(data, self.input_encoding, 'ignore')
+			return str(data, self.input_encoding, 'ignore')
 			
 	def send(self, data):
-		assert isinstance(data, unicode)
+		assert isinstance(data, str)
 		try:
 			encoded = data.encode(self.output_encoding, 'replace')
 		except UnicodeEncodeError:

parse.py

@@ -80,12 +80,14 @@ class Parser(object):
         literal = integer | fnumber | stringlit | listlit | objref
         
         atom << (Optional(minus) + ZeroOrMore(neg) + (propref | literal | bifunction | bexpr | variable | funccall | fileref)).setParseAction(UnaryOp.parse)
+        atom = atom.streamline()
         
         
         # by defining exponentiation as "atom [ ^ factor ]..." instead of "atom [ ^ atom ]...", we get right-to-left exponents, instead of left-to-righ
         # that is, 2^3^2 = 2^(3^2), not (2^3)^2.
         factor = Forward()
         factor << atom + ZeroOrMore( (expop + factor).setParseAction(ArithExp.parse) )
+        factor = factor.streamline()
         
         term = factor + ZeroOrMore( (multop + factor).setParseAction(ArithMul.parse) )
         #term.setParseAction(self.nest)
@@ -114,6 +116,8 @@ class Parser(object):
         assignable = variable | propref | fileref
         assignexpr = Optional(assignable + assign) + boolexpr
         expr << assignexpr.setParseAction(Assignment.parse)
+        expr = expr.streamline()
+
         
         
         """ phase 2:
@@ -167,6 +171,9 @@ class Parser(object):
         block << (exprblock + Optional(ifblock | tryblock | whileblock | forblock) + exprblock)
         lblock << (lexprblock + Optional(iflblock | trylblock | whileblock | forblock) + lexprblock)
 
+        block = block.streamline()
+        lblock = lblock.streamline()
+        
         block.setParseAction(self.nest)
         lblock.setParseAction(self.nest)
 
@@ -197,7 +204,7 @@ class Parser(object):
     def test(self):
         #print(self.parse(u"if (1) #740.xyz + -hello.world; endif"))
         
-		data = unicode(open("test.moo", "r").read(), 'utf-8')
+        data = open("test.moo", "r", encoding="utf-8").read()
         rv = self.parse(data)
         print(rv)
         return rv

pbkdf2.py

@@ -44,7 +44,8 @@ import hmac
 import hashlib
 from struct import Struct
 from operator import xor
-from itertools import izip, starmap
+from itertools import starmap
+import binascii
 
 
 _pack_int = Struct('>I').pack
@@ -52,7 +53,7 @@ _pack_int = Struct('>I').pack
 
 def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc=None):
     """Like :func:`pbkdf2_bin` but returns a hex encoded string."""
-    return pbkdf2_bin(data, salt, iterations, keylen, hashfunc).encode('hex')
+    return str(binascii.hexlify(pbkdf2_bin(data, salt, iterations, keylen, hashfunc)), 'ascii')
 
 
 def pbkdf2_bin(data, salt, iterations=1000, keylen=24, hashfunc=None):
@@ -61,26 +62,29 @@ def pbkdf2_bin(data, salt, iterations=1000, keylen=24, hashfunc=None):
     key of `keylen` bytes.  By default SHA-256 is used as hash function,
     a different hashlib `hashfunc` can be provided.
     """
+
+    bchr = lambda v: bytes((v,))
+
     hashfunc = hashfunc or hashlib.sha256
     mac = hmac.new(data, None, hashfunc)
     def _pseudorandom(x, mac=mac):
         h = mac.copy()
         h.update(x)
-        return map(ord, h.digest())
+        return h.digest()
     buf = []
-    for block in xrange(1, -(-keylen // mac.digest_size) + 1):
+    for block in range(1, -(-keylen // mac.digest_size) + 1):
         rv = u = _pseudorandom(salt + _pack_int(block))
-        for i in xrange(iterations - 1):
+        for i in range(iterations - 1):
-            u = _pseudorandom(''.join(map(chr, u)))
+            u = _pseudorandom(b''.join(map(bchr, u)))
-            rv = starmap(xor, izip(rv, u))
+            rv = starmap(xor, zip(rv, u))
         buf.extend(rv)
-    return ''.join(map(chr, buf))[:keylen]
+    return b''.join(map(bchr, buf))[:keylen]
 
 
 def test():
     failed = []
     def check(data, salt, iterations, keylen, expected):
-        rv = pbkdf2_hex(data, salt, iterations, keylen)
+        rv = pbkdf2_hex(bytes(data, "utf-8"), bytes(salt, "utf-8"), iterations, keylen, hashlib.sha1)
         if rv != expected:
             print('Test failed:')
             print('  Expected:   %s' % expected)