Add admin inspect Glob support (#13328)

* Add admin Glob support

Allow returning multiple files on inspect calls.

```
λ mc admin inspect --json local2/testbucket/nyc-taxi-data-10M.csv.zst/*

...

λ unzip -l inspect.5f0643b2.zip

Archive:  inspect.5f0643b2.zip
  Length      Date    Time    Name
---------  ---------- -----   ----
        0  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
      802  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
      802  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
      802  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
      802  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
---------                     -------
     3208                     8 files
```

Using fully recursive:

```
λ  mc admin inspect local2/testbucket/nyc-taxi-data-10M.csv.zst/**

...

Archive:  inspect.79c261cb.zip
  Length      Date    Time    Name
---------  ---------- -----   ----
        0  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/
        0  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
  3439368  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
      802  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/
        0  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
  3439368  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
      802  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/
        0  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
  3439368  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
      802  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/
        0  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
  3439368  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
      802  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
---------                     -------
601034920                     156 files

```

Furthermore allow `inspect` to do direct decode from `mc`, for example:

```
λ mc admin inspect --json local2/testbucket/nyc-taxi-data-10M.csv.zst/*|inspect -json
Output decrypted to inspect.5f0643b2.zip
```

- Correct error, forward non-EOF errors.
- Add some extra safety. Log FNF when no files.
- Add `xl-meta` zip support.
For `xl-meta` multiple inputs output object with names as key.
Automatically switches `xl-meta` to single-line output when multiple objects.
Add double-star wildcard support to xl-meta input.

Co-authored-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Klaus Post 2021-10-01 11:50:00 -07:00 committed by GitHub
parent 7203d93fb3
commit bc6067d195
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 379 additions and 221 deletions

View File

@ -38,7 +38,7 @@ import (
"strings"
"time"
humanize "github.com/dustin/go-humanize"
"github.com/dustin/go-humanize"
"github.com/gorilla/mux"
"github.com/klauspost/compress/zip"
"github.com/minio/madmin-go"
@ -2144,7 +2144,7 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
// getRawDataer provides an interface for getting raw FS files.
type getRawDataer interface {
GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error
GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error
}
// InspectDataHandler - GET /minio/admin/v3/inspect-data
@ -2177,6 +2177,13 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
}
file = strings.ReplaceAll(file, string(os.PathSeparator), "/")
// Reject attempts to traverse parent or absolute paths.
if strings.Contains(file, "..") || strings.Contains(volume, "..") {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
return
}
var key [32]byte
// MUST use crypto/rand
@ -2214,15 +2221,19 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
zipWriter := zip.NewWriter(encw)
defer zipWriter.Close()
err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time) error {
err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time, isDir bool) error {
// Prefix host+disk
filename = path.Join(host, disk, filename)
if isDir {
filename += "/"
size = 0
}
header, zerr := zip.FileInfoHeader(dummyFileInfo{
name: filename,
size: size,
mode: 0600,
modTime: modtime,
isDir: false,
isDir: isDir,
sys: nil,
})
if zerr != nil {

View File

@ -421,7 +421,7 @@ func TestHealObjectCorrupted(t *testing.T) {
t.Fatalf("Failed to getLatestFileInfo - %v", err)
}
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
}
@ -565,7 +565,7 @@ func TestHealObjectErasure(t *testing.T) {
t.Fatalf("Failed to heal object - %v", err)
}
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
}

View File

@ -18,6 +18,7 @@
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
@ -150,33 +151,45 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
// GetRawData will return all files with a given raw path to the callback.
// Errors are ignored, only errors from the callback are returned.
// For now only direct file paths are supported.
func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error {
found := 0
for _, s := range z.serverPools {
for _, disks := range s.erasureDisks {
for i, disk := range disks {
if disk == OfflineDisk {
continue
}
si, err := disk.StatInfoFile(ctx, volume, file)
stats, err := disk.StatInfoFile(ctx, volume, file, true)
if err != nil {
continue
}
r, err := disk.ReadFileStream(ctx, volume, file, 0, si.Size)
if err != nil {
continue
}
defer r.Close()
did, err := disk.GetDiskID()
if err != nil {
did = fmt.Sprintf("disk-%d", i)
}
err = fn(r, disk.Hostname(), did, pathJoin(volume, file), si.Size, si.ModTime)
if err != nil {
return err
for _, si := range stats {
found++
var r io.ReadCloser
if !si.Dir {
r, err = disk.ReadFileStream(ctx, volume, si.Name, 0, si.Size)
if err != nil {
continue
}
} else {
r = io.NopCloser(bytes.NewBuffer([]byte{}))
}
err = fn(r, disk.Hostname(), did, pathJoin(volume, si.Name), si.Size, si.ModTime, si.Dir)
r.Close()
if err != nil {
return err
}
}
}
}
}
if found == 0 {
return errFileNotFound
}
return nil
}

View File

@ -1508,7 +1508,7 @@ func (fs *FSObjects) RestoreTransitionedObject(ctx context.Context, bucket, obje
// GetRawData returns raw file data to the callback.
// Errors are ignored, only errors from the callback are returned.
// For now only direct file paths are supported.
func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error {
f, err := os.Open(filepath.Join(fs.fsPath, volume, file))
if err != nil {
return nil
@ -1518,5 +1518,5 @@ func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn fun
if err != nil || st.IsDir() {
return nil
}
return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime())
return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime(), st.IsDir())
}

View File

@ -285,9 +285,9 @@ func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi Fi
return d.disk.VerifyFile(ctx, volume, path, fi)
}
func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
if err := d.calcError(); err != nil {
return stat, err
}
return d.disk.StatInfoFile(ctx, volume, path)
return d.disk.StatInfoFile(ctx, volume, path, glob)
}

View File

@ -73,7 +73,7 @@ type StorageAPI interface {
CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
Delete(ctx context.Context, volume string, path string, recursive bool) (err error)
VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error)
StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error)
// Write all data, syncs the data to disk.
// Should be used for smaller payloads.

View File

@ -23,6 +23,7 @@ import (
"encoding/gob"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"net/url"
@ -680,10 +681,11 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st
return toStorageErr(verifyResp.Err)
}
func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
values.Set(storageRESTGlob, fmt.Sprint(glob))
respBody, err := client.call(ctx, storageRESTMethodStatInfoFile, values, nil, -1)
if err != nil {
return stat, err
@ -693,7 +695,19 @@ func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path
if err != nil {
return stat, err
}
err = stat.DecodeMsg(msgpNewReader(respReader))
rd := msgpNewReader(respReader)
for {
var st StatInfo
err = st.DecodeMsg(rd)
if err != nil {
if errors.Is(err, io.EOF) {
err = nil
}
break
}
stat = append(stat, st)
}
return stat, err
}

View File

@ -78,4 +78,5 @@ const (
storageRESTBitrotHash = "bitrot-hash"
storageRESTDiskID = "disk-id"
storageRESTForceDelete = "force-delete"
storageRESTGlob = "glob"
)

View File

@ -1122,13 +1122,16 @@ func (s *storageRESTServer) StatInfoFile(w http.ResponseWriter, r *http.Request)
vars := mux.Vars(r)
volume := vars[storageRESTVolume]
filePath := vars[storageRESTFilePath]
glob := vars[storageRESTGlob]
done := keepHTTPResponseAlive(w)
si, err := s.storage.StatInfoFile(r.Context(), volume, filePath)
stats, err := s.storage.StatInfoFile(r.Context(), volume, filePath, glob == "true")
done(err)
if err != nil {
return
}
msgp.Encode(w, &si)
for _, si := range stats {
msgp.Encode(w, &si)
}
}
// registerStorageRPCRouter - register storage rpc router.
@ -1221,7 +1224,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(httpTraceHdrs(server.WalkDirHandler)).
Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTRecursive)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(httpTraceHdrs(server.StatInfoFile)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTGlob)...)
}
}
}

View File

@ -186,11 +186,11 @@ func testStorageAPIStatInfoFile(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
_, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile)
_, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile, false)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
t.Fatalf("case %v: error: expected: %v, got: %v, err: %v", i+1, expectErr, testCase.expectErr, err)
}
}
}

View File

@ -547,18 +547,18 @@ func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path
return p.storage.ReadAll(ctx, volume, path)
}
func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
defer p.updateStorageMetrics(storageStatInfoFile, volume, path)()
if contextCanceled(ctx) {
return StatInfo{}, ctx.Err()
return nil, ctx.Err()
}
if err = p.checkDiskStale(); err != nil {
return StatInfo{}, err
return nil, err
}
return p.storage.StatInfoFile(ctx, volume, path)
return p.storage.StatInfoFile(ctx, volume, path, glob)
}
func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string) madmin.TraceInfo {

View File

@ -81,6 +81,8 @@ type xlMetaV1Object struct {
type StatInfo struct {
Size int64 `json:"size"` // Size of the object `xl.meta`.
ModTime time.Time `json:"modTime"` // ModTime of the object `xl.meta`.
Name string `json:"name"`
Dir bool `json:"dir"`
}
// ErasureInfo holds erasure coding and bitrot related information.

View File

@ -759,6 +759,18 @@ func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "ModTime")
return
}
case "Name":
z.Name, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
case "Dir":
z.Dir, err = dc.ReadBool()
if err != nil {
err = msgp.WrapError(err, "Dir")
return
}
default:
err = dc.Skip()
if err != nil {
@ -771,10 +783,10 @@ func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) {
}
// EncodeMsg implements msgp.Encodable
func (z StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
func (z *StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 4
// write "Size"
err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
err = en.Append(0x84, 0xa4, 0x53, 0x69, 0x7a, 0x65)
if err != nil {
return
}
@ -793,19 +805,45 @@ func (z StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "ModTime")
return
}
// write "Name"
err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
if err != nil {
return
}
err = en.WriteString(z.Name)
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
// write "Dir"
err = en.Append(0xa3, 0x44, 0x69, 0x72)
if err != nil {
return
}
err = en.WriteBool(z.Dir)
if err != nil {
err = msgp.WrapError(err, "Dir")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z StatInfo) MarshalMsg(b []byte) (o []byte, err error) {
func (z *StatInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
// map header, size 4
// string "Size"
o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
o = append(o, 0x84, 0xa4, 0x53, 0x69, 0x7a, 0x65)
o = msgp.AppendInt64(o, z.Size)
// string "ModTime"
o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
o = msgp.AppendTime(o, z.ModTime)
// string "Name"
o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
o = msgp.AppendString(o, z.Name)
// string "Dir"
o = append(o, 0xa3, 0x44, 0x69, 0x72)
o = msgp.AppendBool(o, z.Dir)
return
}
@ -839,6 +877,18 @@ func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "ModTime")
return
}
case "Name":
z.Name, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
case "Dir":
z.Dir, bts, err = msgp.ReadBoolBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Dir")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@ -852,8 +902,8 @@ func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z StatInfo) Msgsize() (s int) {
s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize
func (z *StatInfo) Msgsize() (s int) {
s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.Name) + 4 + msgp.BoolSize
return
}
@ -1041,40 +1091,11 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
case "Stat":
var zb0002 uint32
zb0002, err = dc.ReadMapHeader()
err = z.Stat.DecodeMsg(dc)
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
for zb0002 > 0 {
zb0002--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
switch msgp.UnsafeString(field) {
case "Size":
z.Stat.Size, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "Stat", "Size")
return
}
case "ModTime":
z.Stat.ModTime, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "Stat", "ModTime")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
}
}
case "Erasure":
err = z.Erasure.DecodeMsg(dc)
if err != nil {
@ -1082,14 +1103,14 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
case "Minio":
var zb0003 uint32
zb0003, err = dc.ReadMapHeader()
var zb0002 uint32
zb0002, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, "Minio")
return
}
for zb0003 > 0 {
zb0003--
for zb0002 > 0 {
zb0002--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err, "Minio")
@ -1111,21 +1132,21 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
}
}
case "Meta":
var zb0004 uint32
zb0004, err = dc.ReadMapHeader()
var zb0003 uint32
zb0003, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, "Meta")
return
}
if z.Meta == nil {
z.Meta = make(map[string]string, zb0004)
z.Meta = make(map[string]string, zb0003)
} else if len(z.Meta) > 0 {
for key := range z.Meta {
delete(z.Meta, key)
}
}
for zb0004 > 0 {
zb0004--
for zb0003 > 0 {
zb0003--
var za0001 string
var za0002 string
za0001, err = dc.ReadString()
@ -1141,16 +1162,16 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
z.Meta[za0001] = za0002
}
case "Parts":
var zb0005 uint32
zb0005, err = dc.ReadArrayHeader()
var zb0004 uint32
zb0004, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err, "Parts")
return
}
if cap(z.Parts) >= int(zb0005) {
z.Parts = (z.Parts)[:zb0005]
if cap(z.Parts) >= int(zb0004) {
z.Parts = (z.Parts)[:zb0004]
} else {
z.Parts = make([]ObjectPartInfo, zb0005)
z.Parts = make([]ObjectPartInfo, zb0004)
}
for za0003 := range z.Parts {
err = z.Parts[za0003].DecodeMsg(dc)
@ -1210,25 +1231,9 @@ func (z *xlMetaV1Object) EncodeMsg(en *msgp.Writer) (err error) {
if err != nil {
return
}
// map header, size 2
// write "Size"
err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
err = z.Stat.EncodeMsg(en)
if err != nil {
return
}
err = en.WriteInt64(z.Stat.Size)
if err != nil {
err = msgp.WrapError(err, "Stat", "Size")
return
}
// write "ModTime"
err = en.Append(0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
if err != nil {
return
}
err = en.WriteTime(z.Stat.ModTime)
if err != nil {
err = msgp.WrapError(err, "Stat", "ModTime")
err = msgp.WrapError(err, "Stat")
return
}
// write "Erasure"
@ -1331,13 +1336,11 @@ func (z *xlMetaV1Object) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.AppendString(o, z.Format)
// string "Stat"
o = append(o, 0xa4, 0x53, 0x74, 0x61, 0x74)
// map header, size 2
// string "Size"
o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
o = msgp.AppendInt64(o, z.Stat.Size)
// string "ModTime"
o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
o = msgp.AppendTime(o, z.Stat.ModTime)
o, err = z.Stat.MarshalMsg(o)
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
// string "Erasure"
o = append(o, 0xa7, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65)
o, err = z.Erasure.MarshalMsg(o)
@ -1408,40 +1411,11 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
case "Stat":
var zb0002 uint32
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
bts, err = z.Stat.UnmarshalMsg(bts)
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
for zb0002 > 0 {
zb0002--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
switch msgp.UnsafeString(field) {
case "Size":
z.Stat.Size, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Stat", "Size")
return
}
case "ModTime":
z.Stat.ModTime, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Stat", "ModTime")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
}
}
case "Erasure":
bts, err = z.Erasure.UnmarshalMsg(bts)
if err != nil {
@ -1449,14 +1423,14 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
case "Minio":
var zb0003 uint32
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
var zb0002 uint32
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Minio")
return
}
for zb0003 > 0 {
zb0003--
for zb0002 > 0 {
zb0002--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err, "Minio")
@ -1478,23 +1452,23 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
}
case "Meta":
var zb0004 uint32
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
var zb0003 uint32
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Meta")
return
}
if z.Meta == nil {
z.Meta = make(map[string]string, zb0004)
z.Meta = make(map[string]string, zb0003)
} else if len(z.Meta) > 0 {
for key := range z.Meta {
delete(z.Meta, key)
}
}
for zb0004 > 0 {
for zb0003 > 0 {
var za0001 string
var za0002 string
zb0004--
zb0003--
za0001, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Meta")
@ -1508,16 +1482,16 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
z.Meta[za0001] = za0002
}
case "Parts":
var zb0005 uint32
zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts)
var zb0004 uint32
zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Parts")
return
}
if cap(z.Parts) >= int(zb0005) {
z.Parts = (z.Parts)[:zb0005]
if cap(z.Parts) >= int(zb0004) {
z.Parts = (z.Parts)[:zb0004]
} else {
z.Parts = make([]ObjectPartInfo, zb0005)
z.Parts = make([]ObjectPartInfo, zb0004)
}
for za0003 := range z.Parts {
bts, err = z.Parts[za0003].UnmarshalMsg(bts)
@ -1552,7 +1526,7 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *xlMetaV1Object) Msgsize() (s int) {
s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize
s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + z.Stat.Msgsize() + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize
if z.Meta != nil {
for za0001, za0002 := range z.Meta {
_ = za0002

View File

@ -46,6 +46,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/console"
"github.com/minio/pkg/env"
"github.com/yargevad/filepathx"
)
const (
@ -1132,7 +1133,7 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
// Check the data path if there is a part with data.
partPath := fmt.Sprintf("part.%d", fi.Parts[0].Number)
dataPath := pathJoin(path, fi.DataDir, partPath)
_, err = s.StatInfoFile(ctx, volume, dataPath)
_, err = s.StatInfoFile(ctx, volume, dataPath, false)
if err != nil {
// Set the inline header, our inlined data is fine.
fi.SetInlineData()
@ -2204,7 +2205,7 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
return nil
}
func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
volumeDir, err := s.getVolDir(volume)
if err != nil {
return stat, err
@ -2221,14 +2222,29 @@ func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat
}
return stat, err
}
filePath := pathJoin(volumeDir, path)
if err := checkPathLength(filePath); err != nil {
return stat, err
var files = []string{pathJoin(volumeDir, path)}
if glob {
files, err = filepathx.Glob(pathJoin(volumeDir, path))
if err != nil {
return nil, err
}
}
st, _ := Lstat(filePath)
if st == nil {
return stat, errPathNotFound
for _, filePath := range files {
if err := checkPathLength(filePath); err != nil {
return stat, err
}
st, _ := Lstat(filePath)
if st == nil {
return stat, errPathNotFound
}
name, err := filepath.Rel(volumeDir, filePath)
if err != nil {
name = filePath
}
if os.PathSeparator != '/' {
name = strings.Replace(name, string(os.PathSeparator), "/", -1)
}
stat = append(stat, StatInfo{ModTime: st.ModTime(), Size: st.Size(), Name: name, Dir: st.IsDir()})
}
return StatInfo{ModTime: st.ModTime(), Size: st.Size()}, nil
return stat, nil
}

View File

@ -1711,7 +1711,7 @@ func TestXLStorageStatInfoFile(t *testing.T) {
}
for i, testCase := range testCases {
_, err := xlStorage.StatInfoFile(context.Background(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile)
_, err := xlStorage.StatInfoFile(context.Background(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile, false)
if err != testCase.expectedErr {
t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
}

View File

@ -114,7 +114,7 @@ func TestIsValidUmaskFile(t *testing.T) {
}
// CheckFile - stat the file.
if _, err := disk.StatInfoFile(context.Background(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile); err != nil {
if _, err := disk.StatInfoFile(context.Background(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile, false); err != nil {
t.Fatalf("Stat failed with %s expected to pass.", err)
}
}

View File

@ -27,10 +27,12 @@ import (
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"github.com/klauspost/compress/zip"
"github.com/minio/cli"
"github.com/tinylib/msgp/msgp"
"github.com/yargevad/filepathx"
)
func main() {
@ -44,9 +46,10 @@ func main() {
USAGE:
{{.Name}} {{if .VisibleFlags}}[FLAGS]{{end}} METAFILES...
Multiple files can be added.
Multiple files can be added. Files ending in ".zip" will be searched for 'xl.meta' files.
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
Double stars means full recursive. testdir/**/xl.meta will search for all xl.meta recursively.
{{if .VisibleFlags}}
GLOBAL FLAGS:
@ -72,43 +75,15 @@ GLOBAL FLAGS:
}
app.Action = func(c *cli.Context) error {
args := c.Args()
if len(args) == 0 {
// If no args, assume xl.meta
args = []string{"xl.meta"}
}
var files []string
for _, pattern := range args {
found, err := filepath.Glob(pattern)
if err != nil {
return err
}
if len(found) == 0 {
return fmt.Errorf("unable to find file %v", pattern)
}
files = append(files, found...)
}
for _, file := range files {
var r io.Reader
switch file {
case "-":
r = os.Stdin
default:
f, err := os.Open(file)
if err != nil {
return err
}
defer f.Close()
r = f
}
ndjson := c.Bool("ndjson")
decode := func(r io.Reader, file string) ([]byte, error) {
b, err := ioutil.ReadAll(r)
if err != nil {
return err
return nil, err
}
b, _, minor, err := checkXL2V1(b)
if err != nil {
return err
return nil, err
}
buf := bytes.NewBuffer(nil)
@ -117,12 +92,12 @@ GLOBAL FLAGS:
case 0:
_, err = msgp.CopyToJSON(buf, bytes.NewBuffer(b))
if err != nil {
return err
return nil, err
}
case 1, 2:
v, b, err := msgp.ReadBytesZC(b)
if err != nil {
return err
return nil, err
}
if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil {
// Read metadata CRC (added in v2, ignore if not found)
@ -131,31 +106,47 @@ GLOBAL FLAGS:
_, err = msgp.CopyToJSON(buf, bytes.NewBuffer(v))
if err != nil {
return err
return nil, err
}
data = b
default:
return fmt.Errorf("unknown metadata version %d", minor)
return nil, fmt.Errorf("unknown metadata version %d", minor)
}
if c.Bool("data") {
b, err := data.json()
if err != nil {
return err
return nil, err
}
buf = bytes.NewBuffer(b)
}
if c.Bool("export") {
file := strings.Map(func(r rune) rune {
switch {
case r >= 'a' && r <= 'z':
return r
case r >= 'A' && r <= 'Z':
return r
case r >= '0' && r <= '9':
return r
case strings.ContainsAny(string(r), "+=-_()!@."):
return r
default:
return '_'
}
}, file)
err := data.files(func(name string, data []byte) {
ioutil.WriteFile(fmt.Sprintf("%s-%s.data", file, name), data, os.ModePerm)
err = ioutil.WriteFile(fmt.Sprintf("%s-%s.data", file, name), data, os.ModePerm)
if err != nil {
fmt.Println(err)
}
})
if err != nil {
return err
return nil, err
}
}
if c.Bool("ndjson") {
fmt.Println(buf.String())
continue
if ndjson {
return buf.Bytes(), nil
}
var msi map[string]interface{}
dec := json.NewDecoder(buf)
@ -163,14 +154,113 @@ GLOBAL FLAGS:
dec.UseNumber()
err = dec.Decode(&msi)
if err != nil {
return err
return nil, err
}
b, err = json.MarshalIndent(msi, "", " ")
if err != nil {
return nil, err
}
return b, nil
}
args := c.Args()
if len(args) == 0 {
// If no args, assume xl.meta
args = []string{"xl.meta"}
}
var files []string
for _, pattern := range args {
if pattern == "-" {
files = append(files, pattern)
continue
}
found, err := filepathx.Glob(pattern)
if err != nil {
return err
}
fmt.Println(string(b))
if len(found) == 0 {
return fmt.Errorf("unable to find file %v", pattern)
}
files = append(files, found...)
}
if len(files) == 0 {
return fmt.Errorf("no files found")
}
multiple := len(files) > 1 || strings.HasSuffix(files[0], ".zip")
if multiple {
ndjson = true
fmt.Println("{")
}
hasWritten := false
for _, file := range files {
var r io.Reader
var sz int64
switch file {
case "-":
r = os.Stdin
default:
f, err := os.Open(file)
if err != nil {
return err
}
if st, err := f.Stat(); err == nil {
sz = st.Size()
}
defer f.Close()
r = f
}
if strings.HasSuffix(file, ".zip") {
zr, err := zip.NewReader(r.(io.ReaderAt), sz)
if err != nil {
return err
}
for _, file := range zr.File {
if !file.FileInfo().IsDir() && strings.HasSuffix(file.Name, "xl.meta") {
r, err := file.Open()
if err != nil {
return err
}
// Quote string...
b, _ := json.Marshal(file.Name)
if hasWritten {
fmt.Print(",\n")
}
fmt.Printf("\t%s: ", string(b))
b, err = decode(r, file.Name)
if err != nil {
return err
}
fmt.Print(string(b))
hasWritten = true
}
}
} else {
if multiple {
// Quote string...
b, _ := json.Marshal(file)
if hasWritten {
fmt.Print(",\n")
}
fmt.Printf("\t%s: ", string(b))
}
b, err := decode(r, file)
if err != nil {
return err
}
hasWritten = true
fmt.Print(string(b))
}
}
fmt.Println("")
if multiple {
fmt.Println("}")
}
return nil
}
err := app.Run(os.Args)

View File

@ -21,10 +21,13 @@ import (
"bufio"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"hash/crc32"
"io"
"io/ioutil"
"log"
"os"
"strings"
@ -34,25 +37,48 @@ import (
var (
key = flag.String("key", "", "decryption string")
//js = flag.Bool("json", false, "expect json input")
js = flag.Bool("json", false, "expect json input from stdin")
)
func main() {
flag.Parse()
if *js {
// Match struct in https://github.com/minio/mc/blob/b3ce21fb72a914f50522358668e6464eb97de8d1/cmd/admin-inspect.go#L135
input := struct {
File string `json:"file"`
Key string `json:"key"`
}{}
got, err := ioutil.ReadAll(os.Stdin)
if err != nil {
fatalErr(err)
}
fatalErr(json.Unmarshal(got, &input))
r, err := os.Open(input.File)
fatalErr(err)
defer r.Close()
dstName := strings.TrimSuffix(input.File, ".enc") + ".zip"
w, err := os.Create(dstName)
fatalErr(err)
defer w.Close()
decrypt(input.Key, r, w)
fmt.Println("Output decrypted to", dstName)
return
}
args := flag.Args()
switch len(flag.Args()) {
case 0:
// Read from stdin, write to stdout.
if *key == "" {
flag.Usage()
fatalErr(errors.New("no key supplied"))
}
decrypt(*key, os.Stdin, os.Stdout)
return
case 1:
r, err := os.Open(args[0])
fatalErr(err)
defer r.Close()
dstName := strings.TrimSuffix(args[0], ".enc") + ".zip"
w, err := os.Create(dstName)
fatalErr(err)
defer w.Close()
if len(*key) == 0 {
reader := bufio.NewReader(os.Stdin)
fmt.Print("Enter Decryption Key: ")
@ -61,18 +87,25 @@ func main() {
// convert CRLF to LF
*key = strings.Replace(text, "\n", "", -1)
}
*key = strings.TrimSpace(*key)
fatalIf(len(*key) != 72, "Unexpected key length: %d, want 72", len(*key))
dstName := strings.TrimSuffix(args[0], ".enc") + ".zip"
w, err := os.Create(dstName)
fatalErr(err)
defer w.Close()
decrypt(*key, r, w)
fmt.Println("Output decrypted to", dstName)
return
default:
flag.Usage()
fatalIf(true, "Only 1 file can be decrypted")
os.Exit(1)
}
}
func decrypt(keyHex string, r io.Reader, w io.Writer) {
keyHex = strings.TrimSpace(keyHex)
fatalIf(len(keyHex) != 72, "Unexpected key length: %d, want 72", len(keyHex))
id, err := hex.DecodeString(keyHex[:8])
fatalErr(err)
key, err := hex.DecodeString(keyHex[8:])

1
go.mod
View File

@ -76,6 +76,7 @@ require (
github.com/valyala/bytebufferpool v1.0.0
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
github.com/yargevad/filepathx v1.0.0
go.etcd.io/etcd/api/v3 v3.5.0-beta.4
go.etcd.io/etcd/client/v3 v3.5.0-beta.4
go.opencensus.io v0.22.5 // indirect

4
go.sum
View File

@ -1030,8 +1030,6 @@ github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8
github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
github.com/minio/minio-go/v7 v7.0.11-0.20210607181445-e162fdb8e584/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA=
github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
github.com/minio/minio-go/v7 v7.0.15-0.20210921183434-174b4c070788 h1:O+/N9vxhoObjuCuQczycuzdG240SoLrrdnyipJ5JJc0=
github.com/minio/minio-go/v7 v7.0.15-0.20210921183434-174b4c070788/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8 h1:+8oaaj9Gm/yJFvR09Mz4iefX7pSYcFw6d6fMfvr1Mow=
github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
github.com/minio/operator v0.0.0-20210812082324-26350f153661 h1:dGAJHpfmhNukFg0M0wDqH+G1OB2YPgZCcT6uv4n9YQk=
@ -1440,6 +1438,8 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=