diff --git a/server/file.c b/server/file.c
index 3e09ddd..5db64cd 100644
--- a/server/file.c
+++ b/server/file.c
@@ -560,7 +560,7 @@
         mode = st.st_mode & (S_ISUID|S_ISGID|S_ISVTX|S_IRWXG);
         mode |= sd_to_mode( sd, owner );
 
-        if (st.st_mode != mode && fchmod( unix_fd, mode ) == -1)
+        if (((st.st_mode ^ mode) & (S_IRWXU|S_IRWXG|S_IRWXO)) && fchmod( unix_fd, mode ) == -1)
         {
             file_set_error();
             return 0;