16 lines
451 B
Bash
Executable file
16 lines
451 B
Bash
Executable file
#! /bin/bash
|
|
|
|
source /workspace/bert/data/utils/config.sh
|
|
|
|
SHARD_COUNT=0
|
|
rm -rf ${TARGET_DIR}/xarg_list.txt
|
|
touch ${TARGET_DIR}/xarg_list.txt
|
|
for file in ${TARGET_DIR}/final_text_files_sharded/*; do
|
|
echo ${SHARD_COUNT} >> ${TARGET_DIR}/xarg_list.txt
|
|
SHARD_COUNT=$((SHARD_COUNT+1))
|
|
done
|
|
|
|
xargs -n 1 --max-procs=${N_PROCS_PREPROCESS} --arg-file=${TARGET_DIR}/xarg_list.txt /workspace/bert/data/utils/preprocessing.sh
|
|
|
|
rm ${TARGET_DIR}/xarg_list.txt
|