futhark-0.20.7: rts/python/values.py
# Start of values.py.
# Hacky parser/reader/writer for values written in Futhark syntax.
# Used for reading stdin when compiling standalone programs with the
# Python code generator.
import numpy as np
import string
import struct
import sys
class ReaderInput:
def __init__(self, f):
self.f = f
self.lookahead_buffer = []
def get_char(self):
if len(self.lookahead_buffer) == 0:
return self.f.read(1)
else:
c = self.lookahead_buffer[0]
self.lookahead_buffer = self.lookahead_buffer[1:]
return c
def unget_char(self, c):
self.lookahead_buffer = [c] + self.lookahead_buffer
def get_chars(self, n):
n1 = min(n, len(self.lookahead_buffer))
s = b''.join(self.lookahead_buffer[:n1])
self.lookahead_buffer = self.lookahead_buffer[n1:]
n2 = n - n1
if n2 > 0:
s += self.f.read(n2)
return s
def peek_char(self):
c = self.get_char()
if c:
self.unget_char(c)
return c
def skip_spaces(f):
c = f.get_char()
while c != None:
if c.isspace():
c = f.get_char()
elif c == b'-':
# May be line comment.
if f.peek_char() == b'-':
# Yes, line comment. Skip to end of line.
while (c != b'\n' and c != None):
c = f.get_char()
else:
break
else:
break
if c:
f.unget_char(c)
def parse_specific_char(f, expected):
got = f.get_char()
if got != expected:
f.unget_char(got)
raise ValueError
return True
def parse_specific_string(f, s):
# This funky mess is intended, and is caused by the fact that if `type(b) ==
# bytes` then `type(b[0]) == int`, but we need to match each element with a
# `bytes`, so therefore we make each character an array element
b = s.encode('utf8')
bs = [b[i:i+1] for i in range(len(b))]
read = []
try:
for c in bs:
parse_specific_char(f, c)
read.append(c)
return True
except ValueError:
for c in read[::-1]:
f.unget_char(c)
raise
def optional(p, *args):
try:
return p(*args)
except ValueError:
return None
def optional_specific_string(f, s):
c = f.peek_char()
# This funky mess is intended, and is caused by the fact that if `type(b) ==
# bytes` then `type(b[0]) == int`, but we need to match each element with a
# `bytes`, so therefore we make each character an array element
b = s.encode('utf8')
bs = [b[i:i+1] for i in range(len(b))]
if c == bs[0]:
return parse_specific_string(f, s)
else:
return False
def sepBy(p, sep, *args):
elems = []
x = optional(p, *args)
if x != None:
elems += [x]
while optional(sep, *args) != None:
x = p(*args)
elems += [x]
return elems
# Assumes '0x' has already been read
def parse_hex_int(f):
s = b''
c = f.get_char()
while c != None:
if c in b'01234556789ABCDEFabcdef':
s += c
c = f.get_char()
elif c == b'_':
c = f.get_char() # skip _
else:
f.unget_char(c)
break
return str(int(s, 16)).encode('utf8') # ugh
def parse_int(f):
s = b''
c = f.get_char()
if c == b'0' and f.peek_char() in b'xX':
c = f.get_char() # skip X
return parse_hex_int(f)
else:
while c != None:
if c.isdigit():
s += c
c = f.get_char()
elif c == b'_':
c = f.get_char() # skip _
else:
f.unget_char(c)
break
if len(s) == 0:
raise ValueError
return s
def parse_int_signed(f):
s = b''
c = f.get_char()
if c == b'-' and f.peek_char().isdigit():
return c + parse_int(f)
else:
if c != b'+':
f.unget_char(c)
return parse_int(f)
def read_str_comma(f):
skip_spaces(f)
parse_specific_char(f, b',')
return b','
def read_str_int(f, s):
skip_spaces(f)
x = int(parse_int_signed(f))
optional_specific_string(f, s)
return x
def read_str_uint(f, s):
skip_spaces(f)
x = int(parse_int(f))
optional_specific_string(f, s)
return x
def read_str_i8(f):
return np.int8(read_str_int(f, 'i8'))
def read_str_i16(f):
return np.int16(read_str_int(f, 'i16'))
def read_str_i32(f):
return np.int32(read_str_int(f, 'i32'))
def read_str_i64(f):
return np.int64(read_str_int(f, 'i64'))
def read_str_u8(f):
return np.uint8(read_str_int(f, 'u8'))
def read_str_u16(f):
return np.uint16(read_str_int(f, 'u16'))
def read_str_u32(f):
return np.uint32(read_str_int(f, 'u32'))
def read_str_u64(f):
return np.uint64(read_str_int(f, 'u64'))
def read_char(f):
skip_spaces(f)
parse_specific_char(f, b'\'')
c = f.get_char()
parse_specific_char(f, b'\'')
return c
def read_str_hex_float(f, sign):
int_part = parse_hex_int(f)
parse_specific_char(f, b'.')
frac_part = parse_hex_int(f)
parse_specific_char(f, b'p')
exponent = parse_int(f)
int_val = int(int_part, 16)
frac_val = float(int(frac_part, 16)) / (16 ** len(frac_part))
exp_val = int(exponent)
total_val = (int_val + frac_val) * (2.0 ** exp_val)
if sign == b'-':
total_val = -1 * total_val
return float(total_val)
def read_str_decimal(f):
skip_spaces(f)
c = f.get_char()
if (c == b'-'):
sign = b'-'
else:
f.unget_char(c)
sign = b''
# Check for hexadecimal float
c = f.get_char()
if (c == '0' and (f.peek_char() in ['x', 'X'])):
f.get_char()
return read_str_hex_float(f, sign)
else:
f.unget_char(c)
bef = optional(parse_int, f)
if bef == None:
bef = b'0'
parse_specific_char(f, b'.')
aft = parse_int(f)
elif optional(parse_specific_char, f, b'.'):
aft = parse_int(f)
else:
aft = b'0'
if (optional(parse_specific_char, f, b'E') or
optional(parse_specific_char, f, b'e')):
expt = parse_int_signed(f)
else:
expt = b'0'
return float(sign + bef + b'.' + aft + b'E' + expt)
def read_str_f16(f):
skip_spaces(f)
try:
parse_specific_string(f, 'f16.nan')
return np.float32(np.nan)
except ValueError:
try:
parse_specific_string(f, 'f16.inf')
return np.float32(np.inf)
except ValueError:
try:
parse_specific_string(f, '-f16.inf')
return np.float32(-np.inf)
except ValueError:
x = read_str_decimal(f)
optional_specific_string(f, 'f16')
return x
def read_str_f32(f):
skip_spaces(f)
try:
parse_specific_string(f, 'f32.nan')
return np.float32(np.nan)
except ValueError:
try:
parse_specific_string(f, 'f32.inf')
return np.float32(np.inf)
except ValueError:
try:
parse_specific_string(f, '-f32.inf')
return np.float32(-np.inf)
except ValueError:
x = read_str_decimal(f)
optional_specific_string(f, 'f32')
return x
def read_str_f64(f):
skip_spaces(f)
try:
parse_specific_string(f, 'f64.nan')
return np.float64(np.nan)
except ValueError:
try:
parse_specific_string(f, 'f64.inf')
return np.float64(np.inf)
except ValueError:
try:
parse_specific_string(f, '-f64.inf')
return np.float64(-np.inf)
except ValueError:
x = read_str_decimal(f)
optional_specific_string(f, 'f64')
return x
def read_str_bool(f):
skip_spaces(f)
if f.peek_char() == b't':
parse_specific_string(f, 'true')
return True
elif f.peek_char() == b'f':
parse_specific_string(f, 'false')
return False
else:
raise ValueError
def read_str_empty_array(f, type_name, rank):
parse_specific_string(f, 'empty')
parse_specific_char(f, b'(')
dims = []
for i in range(rank):
parse_specific_string(f, '[')
dims += [int(parse_int(f))]
parse_specific_string(f, ']')
if np.product(dims) != 0:
raise ValueError
parse_specific_string(f, type_name)
parse_specific_char(f, b')')
return tuple(dims)
def read_str_array_elems(f, elem_reader, type_name, rank):
skip_spaces(f)
try:
parse_specific_char(f, b'[')
except ValueError:
return read_str_empty_array(f, type_name, rank)
else:
xs = sepBy(elem_reader, read_str_comma, f)
skip_spaces(f)
parse_specific_char(f, b']')
return xs
def read_str_array_helper(f, elem_reader, type_name, rank):
def nested_row_reader(_):
return read_str_array_helper(f, elem_reader, type_name, rank-1)
if rank == 1:
row_reader = elem_reader
else:
row_reader = nested_row_reader
return read_str_array_elems(f, row_reader, type_name, rank)
def expected_array_dims(l, rank):
if rank > 1:
n = len(l)
if n == 0:
elem = []
else:
elem = l[0]
return [n] + expected_array_dims(elem, rank-1)
else:
return [len(l)]
def verify_array_dims(l, dims):
if dims[0] != len(l):
raise ValueError
if len(dims) > 1:
for x in l:
verify_array_dims(x, dims[1:])
def read_str_array(f, elem_reader, type_name, rank, bt):
elems = read_str_array_helper(f, elem_reader, type_name, rank)
if type(elems) == tuple:
# Empty array
return np.empty(elems, dtype=bt)
else:
dims = expected_array_dims(elems, rank)
verify_array_dims(elems, dims)
return np.array(elems, dtype=bt)
################################################################################
READ_BINARY_VERSION = 2
# struct format specified at
# https://docs.python.org/2/library/struct.html#format-characters
def mk_bin_scalar_reader(t):
def bin_reader(f):
fmt = FUTHARK_PRIMTYPES[t]['bin_format']
size = FUTHARK_PRIMTYPES[t]['size']
tf = FUTHARK_PRIMTYPES[t]['numpy_type']
return tf(struct.unpack('<' + fmt, f.get_chars(size))[0])
return bin_reader
read_bin_i8 = mk_bin_scalar_reader('i8')
read_bin_i16 = mk_bin_scalar_reader('i16')
read_bin_i32 = mk_bin_scalar_reader('i32')
read_bin_i64 = mk_bin_scalar_reader('i64')
read_bin_u8 = mk_bin_scalar_reader('u8')
read_bin_u16 = mk_bin_scalar_reader('u16')
read_bin_u32 = mk_bin_scalar_reader('u32')
read_bin_u64 = mk_bin_scalar_reader('u64')
read_bin_f16 = mk_bin_scalar_reader('f16')
read_bin_f32 = mk_bin_scalar_reader('f32')
read_bin_f64 = mk_bin_scalar_reader('f64')
read_bin_bool = mk_bin_scalar_reader('bool')
def read_is_binary(f):
skip_spaces(f)
c = f.get_char()
if c == b'b':
bin_version = read_bin_u8(f)
if bin_version != READ_BINARY_VERSION:
panic(1, "binary-input: File uses version %i, but I only understand version %i.\n",
bin_version, READ_BINARY_VERSION)
return True
else:
f.unget_char(c)
return False
FUTHARK_PRIMTYPES = {
'i8': {'binname' : b" i8",
'size' : 1,
'bin_reader': read_bin_i8,
'str_reader': read_str_i8,
'bin_format': 'b',
'numpy_type': np.int8 },
'i16': {'binname' : b" i16",
'size' : 2,
'bin_reader': read_bin_i16,
'str_reader': read_str_i16,
'bin_format': 'h',
'numpy_type': np.int16 },
'i32': {'binname' : b" i32",
'size' : 4,
'bin_reader': read_bin_i32,
'str_reader': read_str_i32,
'bin_format': 'i',
'numpy_type': np.int32 },
'i64': {'binname' : b" i64",
'size' : 8,
'bin_reader': read_bin_i64,
'str_reader': read_str_i64,
'bin_format': 'q',
'numpy_type': np.int64},
'u8': {'binname' : b" u8",
'size' : 1,
'bin_reader': read_bin_u8,
'str_reader': read_str_u8,
'bin_format': 'B',
'numpy_type': np.uint8 },
'u16': {'binname' : b" u16",
'size' : 2,
'bin_reader': read_bin_u16,
'str_reader': read_str_u16,
'bin_format': 'H',
'numpy_type': np.uint16 },
'u32': {'binname' : b" u32",
'size' : 4,
'bin_reader': read_bin_u32,
'str_reader': read_str_u32,
'bin_format': 'I',
'numpy_type': np.uint32 },
'u64': {'binname' : b" u64",
'size' : 8,
'bin_reader': read_bin_u64,
'str_reader': read_str_u64,
'bin_format': 'Q',
'numpy_type': np.uint64 },
'f16': {'binname' : b" f16",
'size' : 2,
'bin_reader': read_bin_f16,
'str_reader': read_str_f16,
'bin_format': 'e',
'numpy_type': np.float16 },
'f32': {'binname' : b" f32",
'size' : 4,
'bin_reader': read_bin_f32,
'str_reader': read_str_f32,
'bin_format': 'f',
'numpy_type': np.float32 },
'f64': {'binname' : b" f64",
'size' : 8,
'bin_reader': read_bin_f64,
'str_reader': read_str_f64,
'bin_format': 'd',
'numpy_type': np.float64 },
'bool': {'binname' : b"bool",
'size' : 1,
'bin_reader': read_bin_bool,
'str_reader': read_str_bool,
'bin_format': 'b',
'numpy_type': bool }
}
def read_bin_read_type(f):
read_binname = f.get_chars(4)
for (k,v) in FUTHARK_PRIMTYPES.items():
if v['binname'] == read_binname:
return k
panic(1, "binary-input: Did not recognize the type '%s'.\n", read_binname)
def numpy_type_to_type_name(t):
for (k,v) in FUTHARK_PRIMTYPES.items():
if v['numpy_type'] == t:
return k
raise Exception('Unknown Numpy type: {}'.format(t))
def read_bin_ensure_scalar(f, expected_type):
dims = read_bin_i8(f)
if dims != 0:
panic(1, "binary-input: Expected scalar (0 dimensions), but got array with %i dimensions.\n", dims)
bin_type = read_bin_read_type(f)
if bin_type != expected_type:
panic(1, "binary-input: Expected scalar of type %s but got scalar of type %s.\n",
expected_type, bin_type)
# ------------------------------------------------------------------------------
# General interface for reading Primitive Futhark Values
# ------------------------------------------------------------------------------
def read_scalar(f, ty):
if read_is_binary(f):
read_bin_ensure_scalar(f, ty)
return FUTHARK_PRIMTYPES[ty]['bin_reader'](f)
return FUTHARK_PRIMTYPES[ty]['str_reader'](f)
def read_array(f, expected_type, rank):
if not read_is_binary(f):
str_reader = FUTHARK_PRIMTYPES[expected_type]['str_reader']
return read_str_array(f, str_reader, expected_type, rank,
FUTHARK_PRIMTYPES[expected_type]['numpy_type'])
bin_rank = read_bin_u8(f)
if bin_rank != rank:
panic(1, "binary-input: Expected %i dimensions, but got array with %i dimensions.\n",
rank, bin_rank)
bin_type_enum = read_bin_read_type(f)
if expected_type != bin_type_enum:
panic(1, "binary-input: Expected %iD-array with element type '%s' but got %iD-array with element type '%s'.\n",
rank, expected_type, bin_rank, bin_type_enum)
shape = []
elem_count = 1
for i in range(rank):
bin_size = read_bin_i64(f)
elem_count *= bin_size
shape.append(bin_size)
bin_fmt = FUTHARK_PRIMTYPES[bin_type_enum]['bin_format']
# We first read the expected number of types into a bytestring,
# then use np.frombuffer. This is because np.fromfile does not
# work on things that are insufficiently file-like, like a network
# stream.
bytes = f.get_chars(elem_count * FUTHARK_PRIMTYPES[expected_type]['size'])
arr = np.frombuffer(bytes, dtype=FUTHARK_PRIMTYPES[bin_type_enum]['numpy_type'])
arr.shape = shape
return arr
if sys.version_info >= (3,0):
input_reader = ReaderInput(sys.stdin.buffer)
else:
input_reader = ReaderInput(sys.stdin)
import re
def read_value(type_desc, reader=input_reader):
"""Read a value of the given type. The type is a string
representation of the Futhark type."""
m = re.match(r'((?:\[\])*)([a-z0-9]+)$', type_desc)
if m:
dims = int(len(m.group(1))/2)
basetype = m.group(2)
assert m and basetype in FUTHARK_PRIMTYPES, "Unknown type: {}".format(type_desc)
if dims > 0:
return read_array(reader, basetype, dims)
else:
return read_scalar(reader, basetype)
def end_of_input(entry, f=input_reader):
skip_spaces(f)
if f.get_char() != b'':
panic(1, "Expected EOF on stdin after reading input for \"%s\".", entry)
def write_value_text(v, out=sys.stdout):
if type(v) == np.uint8:
out.write("%uu8" % v)
elif type(v) == np.uint16:
out.write("%uu16" % v)
elif type(v) == np.uint32:
out.write("%uu32" % v)
elif type(v) == np.uint64:
out.write("%uu64" % v)
elif type(v) == np.int8:
out.write("%di8" % v)
elif type(v) == np.int16:
out.write("%di16" % v)
elif type(v) == np.int32:
out.write("%di32" % v)
elif type(v) == np.int64:
out.write("%di64" % v)
elif type(v) in [bool, np.bool_]:
if v:
out.write("true")
else:
out.write("false")
elif type(v) == np.float16:
if np.isnan(v):
out.write('f16.nan')
elif np.isinf(v):
if v >= 0:
out.write('f16.inf')
else:
out.write('-f16.inf')
else:
out.write("%.6ff16" % v)
elif type(v) == np.float32:
if np.isnan(v):
out.write('f32.nan')
elif np.isinf(v):
if v >= 0:
out.write('f32.inf')
else:
out.write('-f32.inf')
else:
out.write("%.6ff32" % v)
elif type(v) == np.float64:
if np.isnan(v):
out.write('f64.nan')
elif np.isinf(v):
if v >= 0:
out.write('f64.inf')
else:
out.write('-f64.inf')
else:
out.write("%.6ff64" % v)
elif type(v) == np.ndarray:
if np.product(v.shape) == 0:
tname = numpy_type_to_type_name(v.dtype)
out.write('empty({}{})'.format(''.join(['[{}]'.format(d)
for d in v.shape]), tname))
else:
first = True
out.write('[')
for x in v:
if not first: out.write(', ')
first = False
write_value(x, out=out)
out.write(']')
else:
raise Exception("Cannot print value of type {}: {}".format(type(v), v))
type_strs = { np.dtype('int8'): b' i8',
np.dtype('int16'): b' i16',
np.dtype('int32'): b' i32',
np.dtype('int64'): b' i64',
np.dtype('uint8'): b' u8',
np.dtype('uint16'): b' u16',
np.dtype('uint32'): b' u32',
np.dtype('uint64'): b' u64',
np.dtype('float16'): b' f16',
np.dtype('float32'): b' f32',
np.dtype('float64'): b' f64',
np.dtype('bool'): b'bool'}
def construct_binary_value(v):
t = v.dtype
shape = v.shape
elems = 1
for d in shape:
elems *= d
num_bytes = 1 + 1 + 1 + 4 + len(shape) * 8 + elems * t.itemsize
bytes = bytearray(num_bytes)
bytes[0] = np.int8(ord('b'))
bytes[1] = 2
bytes[2] = np.int8(len(shape))
bytes[3:7] = type_strs[t]
for i in range(len(shape)):
bytes[7+i*8:7+(i+1)*8] = np.int64(shape[i]).tobytes()
bytes[7+len(shape)*8:] = np.ascontiguousarray(v).tobytes()
return bytes
def write_value_binary(v, out=sys.stdout):
if sys.version_info >= (3,0):
out = out.buffer
out.write(construct_binary_value(v))
def write_value(v, out=sys.stdout, binary=False):
if binary:
return write_value_binary(v, out=out)
else:
return write_value_text(v, out=out)
# End of values.py.