maxmustermann/armory
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #1874 from MoritzBrueckner/lz4

Browse source 
Support for LZ4 compression
This commit is contained in:
Lubos Lenco 2020-09-21 09:48:20 +02:00 committed by GitHub
parent 5bf0044d50 aab56427d5
commit 8f06f611b5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 184 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
blender/arm/exporter.py
View file

@ -2003,6 +2003,8 @@ class ArmoryExporter:
"""Exports the scene."""
profile_time = time.time()
print('Exporting ' + arm.utils.asset_name(self.scene))
if self.compress_enabled:
print('Scene data will be compressed which might take a while.')
current_frame, current_subframe = self.scene.frame_current, self.scene.frame_subframe

173
blender/arm/lib/lz4.py Normal file
View file

@ -0,0 +1,173 @@
"""
Port of the Iron LZ4 compression module based on
https://github.com/gorhill/lz4-wasm. Original license:
BSD 2-Clause License
Copyright (c) 2018, Raymond Hill
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
import numpy as np
from numpy import uint8, int32, uint32
class LZ4RangeException(Exception):
pass
class LZ4:
hash_table = None
@staticmethod
def encode_bound(size: int) -> int:
return 0 if size > 0x7e000000 else size + (size // 255 | 0) + 16
@staticmethod
def encode(b: bytes) -> bytes:
i_buf: np.ndarray = np.frombuffer(b, dtype=uint8)
i_len = i_buf.size
if i_len >= 0x7e000000:
raise LZ4RangeException("Input buffer is too large")
# "The last match must start at least 12 bytes before end of block"
last_match_pos = i_len - 12
# "The last 5 bytes are always literals"
last_literal_pos = i_len - 5
if LZ4.hash_table is None:
LZ4.hash_table = np.full(shape=65536, fill_value=-65536, dtype=int32)
LZ4.hash_table.fill(-65536)
o_len = LZ4.encode_bound(i_len)
o_buf = np.full(shape=o_len, fill_value=0, dtype=uint8)
i_pos = 0
o_pos = 0
anchor_pos = 0
# Sequence-finding loop
while True:
ref_pos = int32(0)
m_offset = 0
sequence = uint32(i_buf[i_pos] << 8 | i_buf[i_pos + 1] << 16 | i_buf[i_pos + 2] << 24)
# Match-finding loop
while i_pos <= last_match_pos:
# Conversion to uint32 is mandatory to ensure correct
# unsigned right shift (compare with .hx implementation)
sequence = uint32(uint32(sequence) >> uint32(8) | i_buf[i_pos + 3] << 24)
hash_val = (sequence * 0x9e37 & 0xffff) + (uint32(sequence * 0x79b1) >> uint32(16)) & 0xffff
ref_pos = LZ4.hash_table[hash_val]
LZ4.hash_table[hash_val] = i_pos
m_offset = i_pos - ref_pos
if (m_offset < 65536
and i_buf[ref_pos + 0] == (sequence & 0xff)
and i_buf[ref_pos + 1] == ((sequence >> uint32(8)) & 0xff)
and i_buf[ref_pos + 2] == ((sequence >> uint32(16)) & 0xff)
and i_buf[ref_pos + 3] == ((sequence >> uint32(24)) & 0xff)):
break
i_pos += 1
# No match found
if i_pos > last_match_pos:
break
# Match found
l_len = i_pos - anchor_pos
m_len = i_pos
i_pos += 4
ref_pos += 4
while i_pos < last_literal_pos and i_buf[i_pos] == i_buf[ref_pos]:
i_pos += 1
ref_pos += 1
m_len = i_pos - m_len
token = m_len - 4 if m_len < 19 else 15
# Write token, length of literals if needed
if l_len >= 15:
o_buf[o_pos] = 0xf0 | token
o_pos += 1
l = l_len - 15
while l >= 255:
o_buf[o_pos] = 255
o_pos += 1
l -= 255
o_buf[o_pos] = l
o_pos += 1
else:
o_buf[o_pos] = (l_len << 4) | token
o_pos += 1
# Write literals
while l_len > 0:
l_len -= 1
o_buf[o_pos] = i_buf[anchor_pos]
o_pos += 1
anchor_pos += 1
if m_len == 0:
break
# Write offset of match
o_buf[o_pos + 0] = m_offset
o_buf[o_pos + 1] = m_offset >> 8
o_pos += 2
# Write length of match if needed
if m_len >= 19:
l = m_len - 19
while l >= 255:
o_buf[o_pos] = 255
o_pos += 1
l -= 255
o_buf[o_pos] = l
o_pos += 1
anchor_pos = i_pos
# Last sequence is literals only
l_len = i_len - anchor_pos
if l_len >= 15:
o_buf[o_pos] = 0xf0
o_pos += 1
l = l_len - 15
while l >= 255:
o_buf[o_pos] = 255
o_pos += 1
l -= 255
o_buf[o_pos] = l
o_pos += 1
else:
o_buf[o_pos] = l_len << 4
o_pos += 1
while l_len > 0:
l_len -= 1
o_buf[o_pos] = i_buf[anchor_pos]
o_pos += 1
anchor_pos += 1
return np.resize(o_buf, o_pos).tobytes()

2
blender/arm/props.py
View file

@ -77,7 +77,7 @@ def init_properties():
bpy.types.World.arm_loadscreen = BoolProperty(name="Loading Screen", description="Show asset loading progress on published builds", default=True)
bpy.types.World.arm_vsync = BoolProperty(name="VSync", description="Vertical Synchronization", default=True, update=assets.invalidate_compiler_cache)
bpy.types.World.arm_dce = BoolProperty(name="DCE", description="Enable dead code elimination for publish builds", default=True, update=assets.invalidate_compiler_cache)
bpy.types.World.arm_asset_compression = BoolProperty(name="Asset Compression", description="Enable scene data compression", default=False, update=assets.invalidate_compiler_cache)
bpy.types.World.arm_asset_compression = BoolProperty(name="Asset Compression", description="Enable scene data compression with LZ4 when publishing. Warning: This will slow down export!", default=False, update=assets.invalidate_compiler_cache)
bpy.types.World.arm_single_data_file = BoolProperty(name="Single Data File", description="Pack exported meshes and materials into single file", default=False, update=assets.invalidate_compiler_cache)
bpy.types.World.arm_write_config = BoolProperty(name="Write Config", description="Allow this project to be configured at runtime via a JSON file", default=False, update=assets.invalidate_compiler_cache)
bpy.types.World.arm_compiler_inline = BoolProperty(name="Compiler Inline", description="Favor speed over size", default=True, update=assets.invalidate_compiler_cache)

9
blender/arm/utils.py
View file

@ -11,6 +11,7 @@ import numpy as np
import bpy
import arm.lib.armpack
from arm.lib.lz4 import LZ4
import arm.log as log
import arm.make_state as state
@ -23,7 +24,13 @@ class NumpyEncoder(json.JSONEncoder):
def write_arm(filepath, output):
if filepath.endswith('.lz4'):
pass
with open(filepath, 'wb') as f:
packed = arm.lib.armpack.packb(output)
# Prepend packed data size for decoding. Haxe can't unpack
# an unsigned int64 so we use a signed int64 here
f.write(np.int64(LZ4.encode_bound(len(packed))).tobytes())
f.write(LZ4.encode(packed))
else:
if bpy.data.worlds['Arm'].arm_minimize:
with open(filepath, 'wb') as f: