service nfs-config restart

[root@H4-40 ~]

# systemctl status rpc-gssd
● rpc-gssd.service – RPC security service for NFS client and server
Loaded: loaded (/usr/lib/systemd/system/rpc-gssd.service; static; vendor preset: disabled)
Active: active (running) since 四 2017-02-16 13:03:11 CST; 2s ago
Process: 19788 ExecStart=/usr/sbin/rpc.gssd $GSSDARGS (code=exited, status=0/SUCCESS)
Main PID: 19789 (rpc.gssd)
CGroup: /system.slice/rpc-gssd.service
└─19789 /usr/sbin/rpc.gssd

[root@H4-40 ~]

# mount -t nfs4 -o sec=krb5 172.22.4.41:/nas/NAS-00034/fds/krb5 /home/phillips/mount/
mount.nfs4: access denied by server while mounting 172.22.4.41:/nas/NAS-00034/fds/krb5

Feb 16 13:40:51 H4-40 rpc.gssd[19789]: ERROR: No credentials found for connection to server h4-41.falconstor.com.tw

[root@H4-40 ~]

# kinit -k -t /etc/krb5.keytab nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW
kinit: Password incorrect while getting initial credentials

kinit -k -t /etc/krb5.keytab nfs/H4-40.falconstor.com.tw@FALCONSTOR.COM.TW

Feb 16 13:54:12 H4-40 rpc.gssd[19945]: Full hostname for ‘h4-41.falconstor.com.tw’ is ‘h4-41.falconstor.com.tw’
Feb 16 13:54:12 H4-40 rpc.gssd[19945]: Full hostname for ‘h4-40.falconstor.com.tw’ is ‘h4-40.falconstor.com.tw’
Feb 16 13:54:12 H4-40 rpc.gssd[19945]: No key table entry found for H4-40$@FALCONSTOR.COM.TW while getting keytab entry for ‘H4-40$@FALCONSTOR.COM.TW’
Feb 16 13:54:12 H4-40 rpc.gssd[19945]: No key table entry found for H4-40$@FALCONSTOR.COM.TW while getting keytab entry for ‘H4-40$@FALCONSTOR.COM.TW’
Feb 16 13:54:12 H4-40 rpc.gssd[19945]: No key table entry found for root/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW while getting keytab entry for ‘root/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW’
Feb 16 13:54:12 H4-40 rpc.gssd[19945]: Success getting keytab entry for ‘nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW’
Feb 16 13:54:12 H4-40 rpc.gssd[19945]: WARNING: Decrypt integrity check failed while getting initial ticket for principal ‘nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW’ using keytab ‘FILE:/etc/krb5.keytab’
Feb 16 13:54:12 H4-40 rpc.gssd[19945]: ERROR: No credentials found for connection to server h4-41.falconstor.com.tw
Feb 16 13:54:12 H4-40 rpc.gssd[19945]: doing error downcall

[root@H4-40 ~]

# klist -ekt /etc/krb5.keytab | grep h4-40
2 02/15/17 12:29:19 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (AES-256 CTS mode with 96-bit SHA-1 HMAC)
2 02/15/17 12:29:19 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (AES-128 CTS mode with 96-bit SHA-1 HMAC)
2 02/15/17 12:29:19 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Triple DES cbc mode with HMAC/sha1)
2 02/15/17 12:29:19 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (ArcFour with HMAC/md5)
2 02/15/17 12:29:19 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Camellia-256 CTS mode with CMAC)
2 02/15/17 12:29:19 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Camellia-128 CTS mode with CMAC)
2 02/15/17 12:29:19 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (DES with HMAC/sha1)
2 02/15/17 12:29:19 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (DES cbc mode with RSA-MD5)
2 02/15/17 14:35:25 h4-40/falconstor.com.tw@FALCONSTOR.COM.TW (AES-256 CTS mode with 96-bit SHA-1 HMAC)
2 02/15/17 14:35:25 h4-40/falconstor.com.tw@FALCONSTOR.COM.TW (AES-128 CTS mode with 96-bit SHA-1 HMAC)
2 02/15/17 14:35:25 h4-40/falconstor.com.tw@FALCONSTOR.COM.TW (Triple DES cbc mode with HMAC/sha1)
2 02/15/17 14:35:25 h4-40/falconstor.com.tw@FALCONSTOR.COM.TW (ArcFour with HMAC/md5)
2 02/15/17 14:35:25 h4-40/falconstor.com.tw@FALCONSTOR.COM.TW (Camellia-256 CTS mode with CMAC)
2 02/15/17 14:35:25 h4-40/falconstor.com.tw@FALCONSTOR.COM.TW (Camellia-128 CTS mode with CMAC)
2 02/15/17 14:35:25 h4-40/falconstor.com.tw@FALCONSTOR.COM.TW (DES with HMAC/sha1)
2 02/15/17 14:35:25 h4-40/falconstor.com.tw@FALCONSTOR.COM.TW (DES cbc mode with RSA-MD5)
2 02/15/17 16:43:30 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (AES-256 CTS mode with 96-bit SHA-1 HMAC)
2 02/15/17 16:43:30 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (AES-128 CTS mode with 96-bit SHA-1 HMAC)
2 02/15/17 16:43:30 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Triple DES cbc mode with HMAC/sha1)
2 02/15/17 16:43:30 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (ArcFour with HMAC/md5)
2 02/15/17 16:43:30 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Camellia-256 CTS mode with CMAC)
2 02/15/17 16:43:30 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Camellia-128 CTS mode with CMAC)
2 02/15/17 16:43:30 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (DES with HMAC/sha1)
2 02/15/17 16:43:30 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (DES cbc mode with RSA-MD5)
2 02/15/17 17:01:50 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (AES-256 CTS mode with 96-bit SHA-1 HMAC)
2 02/15/17 17:01:50 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (AES-128 CTS mode with 96-bit SHA-1 HMAC)
2 02/15/17 17:01:50 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Triple DES cbc mode with HMAC/sha1)
2 02/15/17 17:01:50 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (ArcFour with HMAC/md5)
2 02/15/17 17:01:50 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Camellia-256 CTS mode with CMAC)
2 02/15/17 17:01:50 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Camellia-128 CTS mode with CMAC)
2 02/15/17 17:01:50 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (DES with HMAC/sha1)
2 02/15/17 17:01:50 host/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (DES cbc mode with RSA-MD5)
2 02/15/17 17:01:59 host/h4-40@FALCONSTOR.COM.TW (AES-256 CTS mode with 96-bit SHA-1 HMAC)
2 02/15/17 17:01:59 host/h4-40@FALCONSTOR.COM.TW (AES-128 CTS mode with 96-bit SHA-1 HMAC)
2 02/15/17 17:01:59 host/h4-40@FALCONSTOR.COM.TW (Triple DES cbc mode with HMAC/sha1)
2 02/15/17 17:01:59 host/h4-40@FALCONSTOR.COM.TW (ArcFour with HMAC/md5)
2 02/15/17 17:01:59 host/h4-40@FALCONSTOR.COM.TW (Camellia-256 CTS mode with CMAC)
2 02/15/17 17:01:59 host/h4-40@FALCONSTOR.COM.TW (Camellia-128 CTS mode with CMAC)
2 02/15/17 17:01:59 host/h4-40@FALCONSTOR.COM.TW (DES with HMAC/sha1)
2 02/15/17 17:01:59 host/h4-40@FALCONSTOR.COM.TW (DES cbc mode with RSA-MD5)
2 02/16/17 09:37:07 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (AES-256 CTS mode with 96-bit SHA-1 HMAC)
2 02/16/17 09:37:07 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (AES-128 CTS mode with 96-bit SHA-1 HMAC)
2 02/16/17 09:37:07 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Triple DES cbc mode with HMAC/sha1)
2 02/16/17 09:37:07 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (ArcFour with HMAC/md5)
2 02/16/17 09:37:07 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Camellia-256 CTS mode with CMAC)
2 02/16/17 09:37:07 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (Camellia-128 CTS mode with CMAC)
2 02/16/17 09:37:07 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (DES with HMAC/sha1)
2 02/16/17 09:37:07 nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW (DES cbc mode with RSA-MD5)

[root@H4-40 ~]

# klist -5fea
Ticket cache: KEYRING:persistent:0:0
Default principal: root@FALCONSTOR.COM.TW

Valid starting Expires Service principal
01/01/70 08:00:00 01/01/70 08:00:00 krb5_ccache_conf_data/fast_avail/krbtgt\/FALCONSTOR.COM.TW\@FALCONSTOR.COM.TW@X-CACHECONF:
Addresses: (none)
02/16/17 14:02:05 02/17/17 14:02:05 krbtgt/FALCONSTOR.COM.TW@FALCONSTOR.COM.TW
Flags: FI, Etype (skey, tkt): AES-256 CTS mode with 96-bit SHA-1 HMAC, AES-256 CTS mode with 96-bit SHA-1 HMAC
Addresses: (none)

[root@H4-41 ~]

# klist
Ticket cache: FILE:/usr/local/vtl/var/krb5cc
Default principal: host/h4-41.falconstor.com.tw@FALCONSTOR.COM.TW

Valid starting Expires Service principal
02/16/17 11:55:59 02/17/17 11:55:59 krbtgt/FALCONSTOR.COM.TW@FALCONSTOR.COM.TW
01/01/70 08:00:00 01/01/70 08:00:00 krb5_ccache_conf_data/fast_avail/krbtgt\/FALCONSTOR.COM.TW\@FALCONSTOR.COM.TW@X-CACHECONF:

RHEL6 need do delete encrypt with etype 25/26

[root@H12-137 ~]

# klist -ek -t /etc/krb5.keytab | grep -n etype | egrep “h12-137|h12-90”
16: 2 02/15/17 12:29:33 host/h12-90.falconstor.com.tw@FALCONSTOR.COM.TW (etype 26)
17: 2 02/15/17 12:29:33 host/h12-90.falconstor.com.tw@FALCONSTOR.COM.TW (etype 25)
160: 2 02/16/17 11:23:26 nfs/h12-137.falconstor.com.tw@FALCONSTOR.COM.TW (etype 26)
161: 2 02/16/17 11:23:26 nfs/h12-137.falconstor.com.tw@FALCONSTOR.COM.TW (etype 25)
168: 2 02/16/17 11:23:37 host/h12-137.falconstor.com.tw@FALCONSTOR.COM.TW (etype 26)
169: 2 02/16/17 11:23:37 host/h12-137.falconstor.com.tw@FALCONSTOR.COM.TW (etype 25)
184: 2 02/16/17 11:59:07 host/h12-90.falconstor.com.tw@FALCONSTOR.COM.TW (etype 26)
185: 2 02/16/17 11:59:07 host/h12-90.falconstor.com.tw@FALCONSTOR.COM.TW (etype 25)
192: 2 02/16/17 11:59:11 nfs/h12-90.falconstor.com.tw@FALCONSTOR.COM.TW (etype 26)
193: 2 02/16/17 11:59:11 nfs/h12-90.falconstor.com.tw@FALCONSTOR.COM.TW (etype 25)

ktutil: rkt /etc/krb5.keytab
ktutil: delent 13 (16-3)
ktutil: delent 14 (17-3)
…
ktutil: wkt /etc/krb5.keytab.new
ktutil: quit
mv /etc/krb5.keytab.new /etc/krb5.keytab #replace it

The RHEL6 NFS client cannot sucessfully mount krb5 if we use FBD v9.0 server as NFS client

Finally, I remove the fdb package and then reinstall nfs-utils for resolve some directory not existing problem.
we not need run “init -k -t /etc/krb5.keytab nfs/h4-40.falconstor.com.tw@FALCONSTOR.COM.TW” in NFS server and NFS client that mean it will auto connect to krb5 server at run “mount -t nfs4 -o sec=krb5 172.22.4.41:/nas/NAS-00034/fds/krb5 /home/phillips/mount”.
Then switch to normal user and run kinit that will see the nfs mount point and access write/read.

kdestroy for exit

— 2019/3/29 —
localhost # /usr/bin/ktutil
ktutil: read_kt /etc/krb5/krb5.keytab
ktutil: list
slot KVNO Principal

1 5 host/localhost@EXAMPLE.COM
ktutil: quit

delete kt tab entry
Example—Removing a Service Principal From a Keytab
In the following example, denver’s host principal is removed from denver’s keytab file.

denver # /usr/sbin/kadmin
kadmin: ktremove host/denver.example.com@EXAMPLE.COM
kadmin: Entry for principal host/denver.example.com@EXAMPLE.COM with kvno 3
removed from keytab WRFILE:/etc/krb5/krb5.keytab.
kadmin: quit

delete principal entry
kadmin: delete_principal nfs/h12-90.falconstor.com.tw@FALCONSTOR.COM.TW

Issue resolved method
check the Kerbert server log file /var/log/krb5kdc.log
Mar 29 16:13:03 FS3147 krb5kdc3742: TGS_REQ (4 etypes {18 17 16 23}) 172.22.16.65: LOOKING_UP_SERVER: authtime 0, nfs/h16-65@FALCONSTOR.COM.TW for nfs/h13-174@FALCONSTOR.COM.TW, Server not found in Kerberos database
Mar 29 16:23:12 FS3147 krb5kdc3742: TGS_REQ (4 etypes {18 17 16 23}) 172.22.16.65: LOOKING_UP_SERVER: authtime 0, nfs/h16-65@FALCONSTOR.COM.TW for nfs/h13-174.falconstor.com.tw@FALCONSTOR.COM.TW, Server not found in Kerberos database

You can found there are two different hostname resolve the h13-174 and h13-174.falconstor.com.tw
because
/etc/hosts format is different
172.22.13.174 H13-174 H13-174.falconstor.com.tw h13-174.falconstor.com.tw <- resolve name as h13-174
172.22.13.174 H13-174.falconstor.com.tw h13-174.falconstor.com.tw H13-174 <- resolve name as h13-174.falconstor.com.tw

workaround:
Add host short name and long name to principal entry

$kadmin

Authenticating as principal root/admin@EXAMPLE.COM with password. Password for root/admin@EXAMPLE.COM: password
kadmin: addprinc -randkey nfs/h21-69.example.com
kadmin: addprinc -randkey host/h21-69.example.com
kadmin: addprinc -randkey nfs/h21-69
kadmin: addprinc -randkey host/h21-69

kadmin: ktadd nfs/h21-69.example.com
kadmin: ktadd host/h21-69.example.com
kadmin: ktadd nfs/h21-69
kadmin: ktadd host/h21-69
kadmin: quit