How do I open a mmap of size of a TB

Question

I need to open a huge memory map. The file is one terabyte.

I however am getting an errno: ENOMEM 12 Cannot allocate memory. I don't get what is holding me up. Requesting the RLIMIT_AS results in the values: 18446744073709551615. Which is enough. My system is also 64 bit so it is not that my virtual memory is too small. ulimit -v is ulimited

I created the data with python using np.lib.format.open_memmap thus it is physically possible. I'm trying to read it in C. Python reading is no problem, numpy.load('terabytearray.npy', mmap_mode='r') works.

---

Here is a minimal example.

Create a numpy array as such:

import numpy as np

shape = (75000, 5000000)
filename = 'datafile.obj'

if __name__ == '__main__':
  arr = np.lib.format.open_memmap(filename, mode='w+', dtype=np.float32, shape=shape)

read it as such:

#include <stdbool.h>
#include <assert.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdint.h>
#include <unistd.h>

#include <sys/time.h>
#include <sys/resource.h>

#include <stdio.h>
#include <errno.h>

typedef enum {
  CNPY_LE, /* little endian (least significant byte to most significant byte) */
  CNPY_BE, /* big endian (most significant byte to least significant byte) */
  CNPY_NE, /* no / neutral endianness (each element is a single byte) */
  /* Host endianness is not supported because it is an incredibly bad idea to
     use it for storage. */
} cnpy_byte_order;

typedef enum {
  CNPY_B = 0, /* We want to use the values as index to the following arrays. */
  CNPY_I1,
  CNPY_I2,
  CNPY_I4,
  CNPY_I8,
  CNPY_U1,
  CNPY_U2,
  CNPY_U4,
  CNPY_U8,
  CNPY_F4,
  CNPY_F8,
  CNPY_C8,
  CNPY_C16,
} cnpy_dtype;

typedef enum {
  CNPY_C_ORDER,       /* C order (row major) */
  CNPY_FORTRAN_ORDER, /* Fortran order (column major) */
} cnpy_flat_order;

typedef enum {
  CNPY_SUCCESS,      /* success */
  CNPY_ERROR_FILE,   /* some error regarding handling of a file */
  CNPY_ERROR_MMAP,   /* some error regarding mmaping a file */
  CNPY_ERROR_FORMAT, /* file format error while reading some file */
} cnpy_status;

#define CNPY_MAX_DIM 4
typedef struct {
  cnpy_byte_order byte_order;
  cnpy_dtype dtype;
  cnpy_flat_order order;
  size_t n_dim;
  size_t dims[CNPY_MAX_DIM];
  char *raw_data;
  size_t data_begin;
  size_t raw_data_size;
} cnpy_array;

cnpy_status cnpy_open(const char * const fn, bool writable, cnpy_array *arr) {
  assert(arr != NULL);

  cnpy_array tmp_arr;

  /* open, mmap, and close the file */
  int fd = open(fn, writable? O_RDWR : O_RDONLY);
  if (fd == -1) {
    return CNPY_ERROR_FILE;
  }
  size_t raw_data_size = (size_t) lseek(fd, 0, SEEK_END);
  lseek(fd, 0, SEEK_SET);
  printf("%lu\n", raw_data_size);
  if (raw_data_size == 0) {
    close(fd); /* no point in checking for errors */
    return CNPY_ERROR_FORMAT;
  }
  if (raw_data_size == SIZE_MAX) {
    /* This is just because the author is too lazy to check for overflow on every pos+1 calculation. */
    close(fd);
    return CNPY_ERROR_FORMAT;
  }

  void *raw_data = mmap(
    NULL,
    raw_data_size,
    PROT_READ | PROT_WRITE,
    writable? MAP_SHARED : MAP_PRIVATE,
    fd,
    0 
  );

  if (raw_data == MAP_FAILED) {
    close(fd);
    return CNPY_ERROR_MMAP;
  }

  if (close(fd) != 0) {
    munmap(raw_data, raw_data_size);
    return CNPY_ERROR_FILE;
  }

  /* parse the file */
  // cnpy_status status = cnpy_parse(raw_data, raw_data_size, &tmp_arr); // library call ignore
  // if (status != CNPY_SUCCESS) {
  //   munmap(raw_data, raw_data_size);
  //   return status;
  // }
  // *arr = tmp_arr;

  return CNPY_SUCCESS;
}

int main(){

  cnpy_array arr = {};
  cnpy_status status = cnpy_open("datafile.obj", false, &arr);

  printf("status %i\n",(int) status);
  if(status != CNPY_SUCCESS){
    printf("failure\n");
    printf("errno %i\n", errno);
  }


    struct rlimit lim;
  printf("getrlimit RLIMIT_AS %s\n", (getrlimit(RLIMIT_AS, &lim) == 0 ? "success" : "failure") );
  printf("lim.rlim_cur %lu\n", lim.rlim_cur );
  printf("lim.rlim_max %lu\n", lim.rlim_max );
  printf("RLIM_INFINITY; %lu\n", RLIM_INFINITY );


  return 0;
}

compile with

gcc -std=c11 -o mmap_testing main.c

I'm using ~quf/cnpy library, I included the relevant parts, to make it work with the numpy stuff.

Answer 1

  void *raw_data = mmap(
    NULL,
    raw_data_size,
    PROT_READ | PROT_WRITE,
    writable? MAP_SHARED : MAP_PRIVATE,
    fd,
    0 
  );

So when writable == false, you are requesting a mapping with PROT_READ | PROT_WRITE and MAP_PRIVATE. That means you want to be able to write to the mapped memory, but not have your writes modify the file. So if you do such a write, a copy will be made (copy-on-write, page by page), but where is it to be kept? It can't be written back to the file on disk, so it can only live in physical memory or swap. Thus this call really is requesting that one terabyte of real memory be allocated or reserved. Which presumably you don't have.

As was discussed in comments, enabling overcommit will allow this to work. In that case the system will pretend it is reserving one terabyte, without actually doing so. If you did write to too much of the memory, then your process would eventually be killed with no possibility to recover (and some other unrelated processes might be killed too).

But it sounds like you don't actually need to write to this memory at all in the writable == false case, so just use PROT_READ by itself. The argument you probably want is something like PROT_READ | (writable ? PROT_WRITE : 0).