inode 没满,但是会随机报磁盘空间不足是啥问题?

huangmingyou

2022-02-11 14:33:40 +08:00

strace mkdir 123456789 看看

duke807

2022-02-11 14:41:31 +08:00

inode 使用的數量和硬盤使用的空間沒有絕對的關係
建立文件系統的時候，使用默認推薦的 inode 數量的話，如果存放超多小文件，最後會導致空間還剩下很多，但是由於沒有多餘 inode 導致無法建立新文件

你说的隨機估計是有其他程序在讀寫同一個文件系統吧

FenixVu

2022-02-11 17:06:31 +08:00

@duke807 我这边确实都是小文件差不多 1000w,问题是我现在磁盘空间也有 inode 也有,我这边是一个 minio 的存储应该只有他在写文件

FenixVu

2022-02-11 17:07:07 +08:00

@huangmingyou 看不太懂

```
execve("/bin/mkdir", ["mkdir", ".minio.sys/buckets/nethospitalse"...], [/* 20 vars */]) = 0
brk(NULL) = 0x12bf000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=35801, ...}) = 0
mmap(NULL, 35801, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fbe711e3000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260Z\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=130224, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbe711e2000
mmap(NULL, 2234080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbe70da5000
mprotect(0x7fbe70dc4000, 2093056, PROT_NONE) = 0
mmap(0x7fbe70fc3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e000) = 0x7fbe70fc3000
mmap(0x7fbe70fc5000, 5856, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fbe70fc5000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\t\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1868984, ...}) = 0
mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbe709db000
mprotect(0x7fbe70b9b000, 2097152, PROT_NONE) = 0
mmap(0x7fbe70d9b000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c0000) = 0x7fbe70d9b000
mmap(0x7fbe70da1000, 14752, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fbe70da1000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpcre.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\25\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=456632, ...}) = 0
mmap(NULL, 2552072, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbe7076b000
mprotect(0x7fbe707d9000, 2097152, PROT_NONE) = 0
mmap(0x7fbe709d9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6e000) = 0x7fbe709d9000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14608, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbe711e1000
mmap(NULL, 2109680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbe70567000
mprotect(0x7fbe7056a000, 2093056, PROT_NONE) = 0
mmap(0x7fbe70769000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fbe70769000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260`\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=138696, ...}) = 0
mmap(NULL, 2212904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbe7034a000
mprotect(0x7fbe70362000, 2093056, PROT_NONE) = 0
mmap(0x7fbe70561000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7fbe70561000
mmap(0x7fbe70563000, 13352, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fbe70563000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbe711e0000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbe711de000
arch_prctl(ARCH_SET_FS, 0x7fbe711de800) = 0
mprotect(0x7fbe70d9b000, 16384, PROT_READ) = 0
mprotect(0x7fbe70561000, 4096, PROT_READ) = 0
mprotect(0x7fbe70769000, 4096, PROT_READ) = 0
mprotect(0x7fbe709d9000, 4096, PROT_READ) = 0
mprotect(0x7fbe70fc3000, 4096, PROT_READ) = 0
mprotect(0x611000, 4096, PROT_READ) = 0
mprotect(0x7fbe711ec000, 4096, PROT_READ) = 0
munmap(0x7fbe711e3000, 35801) = 0
set_tid_address(0x7fbe711dead0) = 62296
set_robust_list(0x7fbe711deae0, 24) = 0
rt_sigaction(SIGRTMIN, {0x7fbe7034fb50, [], SA_RESTORER|SA_SIGINFO, 0x7fbe7035b390}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7fbe7034fbe0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7fbe7035b390}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0x7ffe96852140) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7ffe96852140) = -1 ENOENT (No such file or directory)
brk(NULL) = 0x12bf000
brk(0x12e0000) = 0x12e0000
open("/proc/filesystems", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tr"..., 1024) = 392
read(3, "", 1024) = 0
close(3) = 0
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1668976, ...}) = 0
mmap(NULL, 1668976, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fbe71046000
close(3) = 0
mkdir(".minio.sys/buckets/nethospitalserver/HISREPORT2022011300000686_7a1db02f905b445aa52e6e43134adfc6.xml", 0777) = -1 ENOSPC (No space left on device)
open("/usr/lib/x86_64-linux-gnu/charset.alias", O_RDONLY|O_NOFOLLOW) = -1 ENOENT (No such file or directory)
write(2, "mkdir: ", 7mkdir: ) = 7
write(2, "cannot create directory \342\200\230.mini"..., 129cannot create directory ‘.minio.sys/buckets/nethospitalserver/HISREPORT2022011300000686_7a1db02f905b445aa52e6e43134adfc6.xml’) = 129
write(2, ": No space left on device", 25: No space left on device) = 25
write(2, "\n", 1
) = 1
close(1) = 0
close(2) = 0
exit_group(1) = ?
+++ exited with 1 +++
```

duke807

2022-02-11 17:31:50 +08:00

@FenixVu 空間和 inode 的使用情況，你分別貼出來讓大家看一下，譬如總共多少，已使用多少

查看所用分區空間使用情況的命令是 df
查看 inode 不常用記不住，上網查一下是什麼命令

要進一步排除是否有第三方軟件讀寫的話，linux 也有監控指定路徑讀寫情況的機制

defunct9

2022-02-11 17:42:09 +08:00

df -i

FenixVu

2022-02-11 17:56:55 +08:00

@duke807
Filesystem Type Size Used Avail Use% Mounted on
udev devtmpfs 16G 0 16G 0% /dev
tmpfs tmpfs 3.2G 338M 2.9G 11% /run
/dev/mapper/ubuntu--vg-root ext4 1.5T 700G 745G 49% /
tmpfs tmpfs 16G 4.0K 16G 1% /dev/shm
tmpfs tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs tmpfs 16G 0 16G 0% /sys/fs/cgroup
/dev/sda1 ext2 720M 58M 626M 9% /boot
/home/hisee/.Private ecryptfs 1.5T 700G 745G 49% /home/hisee
overlay overlay 1.5T 700G 745G 49%

Filesystem Inodes IUsed IFree IUse% Mounted on
udev 4110911 430 4110481 1% /dev
tmpfs 4116178 865 4115313 1% /run
/dev/mapper/ubuntu--vg-root 100425728 35748318 64677410 36% /
tmpfs 4116178 2 4116176 1% /dev/shm
tmpfs 4116178 3 4116175 1% /run/lock
tmpfs 4116178 16 4116162 1% /sys/fs/cgroup
/dev/sda1 46848 299 46549 1% /boot
/home/hisee/.Private 100425728 35748318 64677410 36% /home/hisee
overlay 100425728 35748318 64677410 36% /var/lib/docker/overlay2/8bdfd3b8eb24e7422778b6e6ecc13a900bfa12bbad903fc283694d0148b511ca/merged

feedcode

2022-02-11 19:35:59 +08:00

遇到了哈希冲突，
ext4 uses half_md4 as a default hashing-mechanism. If I interpret my google-results correctly, this uses the md4-hash algorithm, but strips it to 32 bits.

https://blog.merovius.de/2013/10/20/ext4-mysterious-no-space-left-on.html

zhoudaiyu

2022-02-11 19:56:52 +08:00

@feedcode #8
@FenixVu #3 lz 贴个"lsblk -f"的结果呗

zhoudaiyu

2022-02-11 19:58:18 +08:00

@zhoudaiyu #9 看到了 lz 贴的 df 的结果了，抱歉

zhoudaiyu

2022-02-11 19:58:56 +08:00

@feedcode #8 这都行，学到了！

liuxu

2022-02-11 20:20:02 +08:00

@FenixVu #4

结合楼主之前的帖子和 strace ，是因为你测试的目录是 minio 所管理的空间，然后 minio 服务器使用的.minio.sys
/目录磁盘爆了

mkdir(".minio.sys/buckets/nethospitalserver/HISREPORT2022011300000686_7a1db02f905b445aa52e6e43134adfc6.xml", 0777) = -1 ENOSPC (No space left on device)
open("/usr/lib/x86_64-linux-gnu/charset.alias", O_RDONLY|O_NOFOLLOW) = -1 ENOENT (No such file or directory)
write(2, "mkdir: ", 7mkdir: ) = 7
write(2, "cannot create directory \342\200\230.mini"..., 129cannot create directory ‘.minio.sys/buckets/nethospitalserver/HISREPORT2022011300000686_7a1db02f905b445aa52e6e43134adfc6.xml’) = 129
write(2, ": No space left on device", 25: No space left on device) = 25
write(2, "\n", 1
) = 1

创建.minio.sys/buckets/nethospitalserver/HISREPORT2022011300000686_7a1db02f905b445aa52e6e43134adfc6.xml 没空间了

给你看看我的 strace

liuxu@liuxu-Inspiron-7559:/tmp$ strace mkdir 123456789
execve("/usr/bin/mkdir", ["mkdir", "123456789"], 0x7ffd98b5c8c8 /* 60 vars */) = 0
brk(NULL) = 0x5568f5d55000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffc7dca4820) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
...
...
access("/etc/selinux/config", F_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=14537584, ...}) = 0
mmap(NULL, 14537584, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff0d0e1e000
close(3) = 0
mkdir("123456789", 0777) = 0
close(1) = 0
close(2) = 0
exit_group(0) = ?
+++ exited with 0 +++

最下面 access("/etc/selinux/config", F_OK) 后直接 mkdir("123456789", 0777) ，楼主的连 mkdir("123456789", 0777)都没执行

楼上的大哥们虽然都是好心，但是别乱指路啊

duke807

2022-02-11 20:46:42 +08:00

沒看出來樓主 .minio.sys 目錄在哪個路徑，應該是在 overlay 管轄的 home 分區

不知道 #8 提到的問題是否會影響 ext4 之上的 overlay ，我覺得應該會影響

反正 #8 提供的信息對我有幫助，我就給讚了
反而 #12 的觀點我不認同

liuxu

2022-02-11 21:54:51 +08:00

@duke807 #13
8 楼的也有可能 ext4 问题，创建文件夹对于 minio 对象系统来说可能就是只创建一个 xml 文件记录一下，而不会调用 mkdir 123456789

但如果是 hash 导致的问题，很难解释通创建 123456789 失败，而创建 123456789a 却可以间歇成功
应该是不管是 123456789 还是 123456789a ，都一直失败或都间歇失败

liuxu

2022-02-11 22:00:29 +08:00

要是 8 楼的还不行，看看 inode 是不是没用完

$ df . | sed -n '$s/ .*//p' | xargs sudo dumpe2fs -h

documentzhangx66

2022-02-12 02:59:31 +08:00

这个问题其实很简单。

你这盘又没多大，目前不确定到底是 ext4 还是 minio 的问题，那就排除法嘛。

先做一个同样大小、同样 ext4 参数的盘，把 ext4 dd 过去，用同样的系统挂载，再创建文件，看看会不会失败。

如果失败，说明是 ext4 的问题，如果成功，那就说明是 minio 的问题。

接下来就很简单了。

如果是 ext4 的问题，调试 ext4 ；如果是 minio 的问题，调试 minio 。

levinit

2022-02-12 08:54:54 +08:00

@feedcode 学习了留个心眼

FenixVu

2022-02-12 10:36:39 +08:00

@feedcode 我开始也考虑过是哈希问题,但是我想的方向是底层硬盘某一块满了,去查看后发现并没有,没往文件系统这块想,感谢你分享的文章看完后发现只能更换文件系统类型了