msdosfs: fixes for Undefined Behavior.

[pfg]

Summary

These were found by the Undefined Behaviour GsoC project at NetBSD:

Do not change signedness bit with left shift.
While there avoid signed integer overflow.
Address both issues with using unsigned type.

msdosfs_fat.c:512:42, left shift of 1 by 31 places cannot be represented
in type 'int'
msdosfs_fat.c:521:44, left shift of 1 by 31 places cannot be represented
in type 'int'
msdosfs_fat.c:744:14, left shift of 1 by 31 places cannot be represented
in type 'int'
msdosfs_fat.c:744:24, signed integer overflow: -2147483648 - 1 cannot be
represented in type 'int [20]'
msdosfs_fat.c:840:13, left shift of 1 by 31 places cannot be represented
in type 'int'
msdosfs_fat.c:840:36, signed integer overflow: -2147483648 - 1 cannot be
represented in type 'int [20]'

Detected with micro-UBSan in the user mode.

Hinted from: NetBSD

[delphij]

LGTM. Thanks!

[pfg]

This was committed as r337456.

sys/fs/msdosfs/msdosfs_fat.c

usemap_alloc(struct msdosfsmount *pmp, u_long cn)
 
         KASSERT(cn <= pmp->pm_maxcluster, ("cn too large %lu %lu", cn,
             pmp->pm_maxcluster));
         KASSERT((pmp->pm_flags & MSDOSFSMNT_RONLY) == 0,
             ("usemap_alloc on ro msdosfs mount"));
         KASSERT((pmp->pm_inusemap[cn / N_INUSEBITS] & (1 << (cn % N_INUSEBITS)))
             == 0, ("Allocating used sector %ld %ld %x", cn, cn % N_INUSEBITS,
                 (unsigned)pmp->pm_inusemap[cn / N_INUSEBITS]));
-        pmp->pm_inusemap[cn / N_INUSEBITS] |= 1 << (cn % N_INUSEBITS);
+        pmp->pm_inusemap[cn / N_INUSEBITS] |= 1U << (cn % N_INUSEBITS);
         KASSERT(pmp->pm_freeclustercount > 0, ("usemap_alloc: too little"));
         pmp->pm_freeclustercount--;
         pmp->pm_flags |= MSDOSFS_FSIMOD;
 }
 
 static __inline void
 usemap_free(struct msdosfsmount *pmp, u_long cn)
 {
 
         MSDOSFS_ASSERT_MP_LOCKED(pmp);
 
         KASSERT(cn <= pmp->pm_maxcluster, ("cn too large %lu %lu", cn,
             pmp->pm_maxcluster));
         KASSERT((pmp->pm_flags & MSDOSFSMNT_RONLY) == 0,
             ("usemap_free on ro msdosfs mount"));
         pmp->pm_freeclustercount++;
         pmp->pm_flags |= MSDOSFS_FSIMOD;
         KASSERT((pmp->pm_inusemap[cn / N_INUSEBITS] & (1 << (cn % N_INUSEBITS)))
             != 0, ("Freeing unused sector %ld %ld %x", cn, cn % N_INUSEBITS,
                 (unsigned)pmp->pm_inusemap[cn / N_INUSEBITS]));
-        pmp->pm_inusemap[cn / N_INUSEBITS] &= ~(1 << (cn % N_INUSEBITS));
+        pmp->pm_inusemap[cn / N_INUSEBITS] &= ~(1U << (cn % N_INUSEBITS));
 }
 
 int
 clusterfree(struct msdosfsmount *pmp, u_long cluster, u_long *oldcnp)
 {
         int error;
         u_long oldcn;
 
if (start) {
                 len = 0;
 
         newst = pmp->pm_nxtfree;
         foundl = 0;
 
         for (cn = newst; cn <= pmp->pm_maxcluster;) {
                 idx = cn / N_INUSEBITS;
                 map = pmp->pm_inusemap[idx];
-                map |= (1 << (cn % N_INUSEBITS)) - 1;
+                map |= (1U << (cn % N_INUSEBITS)) - 1;
                 if (map != FULL_RUN) {
                         cn = idx * N_INUSEBITS + ffs(map ^ FULL_RUN) - 1;
                         if ((l = chainlength(pmp, cn, count)) >= count)
                                 return (chainalloc(pmp, cn, count, fillwith, retcluster, got));
                         if (l > foundl) {
                                 foundcn = cn;
                                 foundl = l;
                         }
                         cn += l + 1;
                         continue;
                 }
                 cn += N_INUSEBITS - cn % N_INUSEBITS;
         }
         for (cn = 0; cn < newst;) {
                 idx = cn / N_INUSEBITS;
                 map = pmp->pm_inusemap[idx];
-                map |= (1 << (cn % N_INUSEBITS)) - 1;
+                map |= (1U << (cn % N_INUSEBITS)) - 1;
                 if (map != FULL_RUN) {
                         cn = idx * N_INUSEBITS + ffs(map ^ FULL_RUN) - 1;
                         if ((l = chainlength(pmp, cn, count)) >= count)
                                 return (chainalloc(pmp, cn, count, fillwith, retcluster, got));
                         if (l > foundl) {
                                 foundcn = cn;
                                 foundl = l;
                         }
#endif
                 } else if (readcn == CLUST_FREE)
                         usemap_free(pmp, cn);
         }
         if (bp != NULL)
                 brelse(bp);
 
         for (cn = pmp->pm_maxcluster + 1; cn < (pmp->pm_maxcluster +
             N_INUSEBITS) / N_INUSEBITS; cn++)
-                pmp->pm_inusemap[cn / N_INUSEBITS] |= 1 << (cn % N_INUSEBITS);
+                pmp->pm_inusemap[cn / N_INUSEBITS] |= 1U << (cn % N_INUSEBITS);
 
         return (0);
 }
 
 /*
  * Allocate a new cluster and chain it onto the end of the file.
  *
  * dep   - the file to extend