You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mynewt.apache.org by cc...@apache.org on 2016/04/27 01:37:44 UTC
incubator-mynewt-core git commit: MYNEWT-75 NFFS - Garbage collect on
OOM.
Repository: incubator-mynewt-core
Updated Branches:
refs/heads/develop c12dbb96b -> 0970df86c
MYNEWT-75 NFFS - Garbage collect on OOM.
Project: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/commit/0970df86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/tree/0970df86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/diff/0970df86
Branch: refs/heads/develop
Commit: 0970df86c887e2ad1d9cd3984ea9c2b3091d6fba
Parents: c12dbb9
Author: Christopher Collins <cc...@apache.org>
Authored: Fri Apr 22 21:12:19 2016 -0700
Committer: Christopher Collins <cc...@apache.org>
Committed: Tue Apr 26 16:35:52 2016 -0700
----------------------------------------------------------------------
fs/nffs/src/nffs_block.c | 31 ++++++++++--
fs/nffs/src/nffs_file.c | 5 +-
fs/nffs/src/nffs_inode.c | 25 ++++++++++
fs/nffs/src/nffs_misc.c | 72 +++++++++++++++++++++++++++-
fs/nffs/src/nffs_priv.h | 3 ++
fs/nffs/src/nffs_write.c | 2 +-
fs/nffs/src/test/arch/sim/nffs_test.c | 75 ++++++++++++++++++++++++++++--
7 files changed, 202 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/blob/0970df86/fs/nffs/src/nffs_block.c
----------------------------------------------------------------------
diff --git a/fs/nffs/src/nffs_block.c b/fs/nffs/src/nffs_block.c
index 2186444..126a58c 100644
--- a/fs/nffs/src/nffs_block.c
+++ b/fs/nffs/src/nffs_block.c
@@ -39,10 +39,35 @@ nffs_block_entry_alloc(void)
}
void
-nffs_block_entry_free(struct nffs_hash_entry *entry)
+nffs_block_entry_free(struct nffs_hash_entry *block_entry)
{
- assert(nffs_hash_id_is_block(entry->nhe_id));
- os_memblock_put(&nffs_block_entry_pool, entry);
+ assert(nffs_hash_id_is_block(block_entry->nhe_id));
+ os_memblock_put(&nffs_block_entry_pool, block_entry);
+}
+
+/**
+ * Allocates a block entry. If allocation fails due to memory exhaustion,
+ * garbage collection is performed and the allocation is retried. This
+ * process is repeated until allocation is successful or all areas have been
+ * garbage collected.
+ *
+ * @param out_block_entry On success, the address of the allocated
+ * block gets written here.
+ *
+ * @return 0 on successful allocation;
+ * FS_ENOMEM on memory exhaustion;
+ * other nonzero on garbage collection error.
+ */
+int
+nffs_block_entry_reserve(struct nffs_hash_entry **out_block_entry)
+{
+ int rc;
+
+ do {
+ *out_block_entry = nffs_block_entry_alloc();
+ } while (nffs_misc_gc_if_oom(*out_block_entry, &rc));
+
+ return rc;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/blob/0970df86/fs/nffs/src/nffs_file.c
----------------------------------------------------------------------
diff --git a/fs/nffs/src/nffs_file.c b/fs/nffs/src/nffs_file.c
index 40741df..6745df7 100644
--- a/fs/nffs/src/nffs_file.c
+++ b/fs/nffs/src/nffs_file.c
@@ -75,9 +75,8 @@ nffs_file_new(struct nffs_inode_entry *parent, const char *filename,
uint8_t area_idx;
int rc;
- inode_entry = nffs_inode_entry_alloc();
- if (inode_entry == NULL) {
- rc = FS_ENOMEM;
+ rc = nffs_inode_entry_reserve(&inode_entry);
+ if (rc != 0) {
goto err;
}
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/blob/0970df86/fs/nffs/src/nffs_inode.c
----------------------------------------------------------------------
diff --git a/fs/nffs/src/nffs_inode.c b/fs/nffs/src/nffs_inode.c
index ffef547..5beb87d 100644
--- a/fs/nffs/src/nffs_inode.c
+++ b/fs/nffs/src/nffs_inode.c
@@ -59,6 +59,31 @@ nffs_inode_entry_free(struct nffs_inode_entry *inode_entry)
}
}
+/**
+ * Allocates a inode entry. If allocation fails due to memory exhaustion,
+ * garbage collection is performed and the allocation is retried. This
+ * process is repeated until allocation is successful or all areas have been
+ * garbage collected.
+ *
+ * @param out_inode_entry On success, the address of the allocated
+ * inode gets written here.
+ *
+ * @return 0 on successful allocation;
+ * FS_ENOMEM on memory exhaustion;
+ * other nonzero on garbage collection error.
+ */
+int
+nffs_inode_entry_reserve(struct nffs_inode_entry **out_inode_entry)
+{
+ int rc;
+
+ do {
+ *out_inode_entry = nffs_inode_entry_alloc();
+ } while (nffs_misc_gc_if_oom(*out_inode_entry, &rc));
+
+ return rc;
+}
+
uint32_t
nffs_inode_disk_size(const struct nffs_inode *inode)
{
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/blob/0970df86/fs/nffs/src/nffs_misc.c
----------------------------------------------------------------------
diff --git a/fs/nffs/src/nffs_misc.c b/fs/nffs/src/nffs_misc.c
index e3e1dc7..8dbff0a 100644
--- a/fs/nffs/src/nffs_misc.c
+++ b/fs/nffs/src/nffs_misc.c
@@ -6,7 +6,7 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
@@ -90,6 +90,76 @@ nffs_misc_validate_scratch(void)
}
/**
+ * Performs a garbage cycle to free up memory, if necessary. This function
+ * should be called repeatedly until either:
+ * o The subsequent allocation is successful, or
+ * o Garbage collection is not successfully performed (indicated by a return
+ * code other than FS_EAGAIN).
+ *
+ * This function determines if garbage collection is necessary by inspecting
+ * the value of the supplied "resource" parameter. If resource is null, that
+ * implies that allocation failed.
+ *
+ * This function will not initiate garbage collection if all areas have already
+ * been collected in an attempt to free memory for the allocation in question.
+ *
+ * @param resource The result of the allocation attempt; null
+ * implies that garbage collection is
+ * necessary.
+ * @param out_rc The status of this operation gets written here.
+ * 0: garbage collection was successful or
+ * unnecessary.
+ * FS_EFULL: Garbage collection was not
+ * performed because all areas have
+ * already been collected.
+ * Other nonzero: garbage collection failed.
+ *
+ * @return FS_EAGAIN if garbage collection was
+ * successfully performed and the allocation
+ * should be retried;
+ * Other value if the allocation should not be
+ * retried; the value of the out_rc parameter
+ * indicates whether allocation was successful
+ * or there was an error.
+ */
+int
+nffs_misc_gc_if_oom(void *resource, int *out_rc)
+{
+ /**
+ * Keeps track of the number of repeated garbage collection cycles.
+ * Necessary for ensuring GC stops after all areas have been collected.
+ */
+ static uint8_t total_gc_cycles;
+
+ if (resource != NULL) {
+ /* Allocation succeeded. Reset cycle count in preparation for the next
+ * allocation failure.
+ */
+ total_gc_cycles = 0;
+ *out_rc = 0;
+ return 0;
+ }
+
+ /* If every area has already been garbage collected, there is nothing else
+ * that can be done ("- 1" to account for the scratch area).
+ */
+ if (total_gc_cycles >= nffs_num_areas - 1) {
+ *out_rc = FS_ENOMEM;
+ return 0;
+ }
+
+ /* Attempt a garbage collection on the next area. */
+ *out_rc = nffs_gc(NULL);
+ total_gc_cycles++;
+ if (*out_rc != 0) {
+ return 0;
+ }
+
+ /* Indicate that garbage collection was successfully performed. */
+ return 1;
+}
+
+/**
* Reserves the specified number of bytes within the specified area.
*
* @param area_idx The index of the area to reserve from.
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/blob/0970df86/fs/nffs/src/nffs_priv.h
----------------------------------------------------------------------
diff --git a/fs/nffs/src/nffs_priv.h b/fs/nffs/src/nffs_priv.h
index 49b72ea..2168289 100644
--- a/fs/nffs/src/nffs_priv.h
+++ b/fs/nffs/src/nffs_priv.h
@@ -254,6 +254,7 @@ int nffs_area_find_corrupt_scratch(uint16_t *out_good_idx,
/* @block */
struct nffs_hash_entry *nffs_block_entry_alloc(void);
void nffs_block_entry_free(struct nffs_hash_entry *entry);
+int nffs_block_entry_reserve(struct nffs_hash_entry **out_block_entry);
int nffs_block_read_disk(uint8_t area_idx, uint32_t area_offset,
struct nffs_disk_block *out_disk_block);
int nffs_block_write_disk(const struct nffs_disk_block *disk_block,
@@ -355,6 +356,7 @@ int nffs_hash_init(void);
/* @inode */
struct nffs_inode_entry *nffs_inode_entry_alloc(void);
void nffs_inode_entry_free(struct nffs_inode_entry *inode_entry);
+int nffs_inode_entry_reserve(struct nffs_inode_entry **out_inode_entry);
int nffs_inode_calc_data_length(struct nffs_inode_entry *inode_entry,
uint32_t *out_len);
int nffs_inode_data_len(struct nffs_inode_entry *inode_entry,
@@ -401,6 +403,7 @@ int nffs_inode_unlink_from_ram_corrupt_ok(struct nffs_inode *inode,
int nffs_inode_unlink(struct nffs_inode *inode);
/* @misc */
+int nffs_misc_gc_if_oom(void *resource, int *out_rc);
int nffs_misc_reserve_space(uint16_t space,
uint8_t *out_area_idx, uint32_t *out_area_offset);
int nffs_misc_set_num_areas(uint8_t num_areas);
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/blob/0970df86/fs/nffs/src/nffs_write.c
----------------------------------------------------------------------
diff --git a/fs/nffs/src/nffs_write.c b/fs/nffs/src/nffs_write.c
index 8888016..95927cd 100644
--- a/fs/nffs/src/nffs_write.c
+++ b/fs/nffs/src/nffs_write.c
@@ -219,7 +219,7 @@ nffs_write_append(struct nffs_cache_inode *cache_inode, const void *data,
uint8_t area_idx;
int rc;
- entry = nffs_block_entry_alloc();
+ rc = nffs_block_entry_reserve(&entry);
if (entry == NULL) {
return FS_ENOMEM;
}
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-core/blob/0970df86/fs/nffs/src/test/arch/sim/nffs_test.c
----------------------------------------------------------------------
diff --git a/fs/nffs/src/test/arch/sim/nffs_test.c b/fs/nffs/src/test/arch/sim/nffs_test.c
index 8078ea0..bf97ef8 100644
--- a/fs/nffs/src/test/arch/sim/nffs_test.c
+++ b/fs/nffs/src/test/arch/sim/nffs_test.c
@@ -269,7 +269,7 @@ nffs_test_util_create_file_blocks(const char *filename,
static void
nffs_test_util_create_file(const char *filename, const char *contents,
- int contents_len)
+ int contents_len)
{
struct nffs_test_block_desc block;
@@ -281,7 +281,7 @@ nffs_test_util_create_file(const char *filename, const char *contents,
static void
nffs_test_util_append_file(const char *filename, const char *contents,
- int contents_len)
+ int contents_len)
{
struct fs_file *file;
int rc;
@@ -298,7 +298,7 @@ nffs_test_util_append_file(const char *filename, const char *contents,
static void
nffs_test_copy_area(const struct nffs_area_desc *from,
- const struct nffs_area_desc *to)
+ const struct nffs_area_desc *to)
{
void *buf;
int rc;
@@ -2495,6 +2495,74 @@ TEST_CASE(nffs_test_split_file)
nffs_test_assert_system(expected_system, area_descs_two);
}
+TEST_CASE(nffs_test_gc_on_oom)
+{
+ int rc;
+
+ /*** Setup. */
+ /* Ensure all areas are the same size. */
+ static const struct nffs_area_desc area_descs_two[] = {
+ { 0x00000000, 16 * 1024 },
+ { 0x00004000, 16 * 1024 },
+ { 0x00008000, 16 * 1024 },
+ { 0, 0 },
+ };
+
+ rc = nffs_format(area_descs_two);
+ TEST_ASSERT_FATAL(rc == 0);
+
+ /* Leak block entries until only four are left. */
+ /* XXX: This is ridiculous. Need to fix nffs configuration so that the
+ * caller passes a config object rather than writing to a global variable.
+ */
+ while (nffs_block_entry_pool.mp_num_free != 4) {
+ nffs_block_entry_alloc();
+ }
+
+ /*** Write 4 data blocks. */
+ struct nffs_test_block_desc blocks[4] = { {
+ .data = "1",
+ .data_len = 1,
+ }, {
+ .data = "2",
+ .data_len = 1,
+ }, {
+ .data = "3",
+ .data_len = 1,
+ }, {
+ .data = "4",
+ .data_len = 1,
+ } };
+
+ nffs_test_util_create_file_blocks("/myfile.txt", blocks, 4);
+
+ TEST_ASSERT_FATAL(nffs_block_entry_pool.mp_num_free == 0);
+
+ /* Attempt another one-byte write. This should trigger a garbage
+ * collection cycle, resulting in the four blocks being collated. The
+ * fifth write consumes an additional block, resulting in 2 out of 4 blocks
+ * in use.
+ */
+ nffs_test_util_append_file("/myfile.txt", "5", 1);
+
+ TEST_ASSERT_FATAL(nffs_block_entry_pool.mp_num_free == 2);
+
+ struct nffs_test_file_desc *expected_system =
+ (struct nffs_test_file_desc[]) { {
+ .filename = "",
+ .is_dir = 1,
+ .children = (struct nffs_test_file_desc[]) { {
+ .filename = "myfile.txt",
+ .contents = "12345",
+ .contents_len = 5,
+ }, {
+ .filename = NULL,
+ } },
+ } };
+
+ nffs_test_assert_system(expected_system, area_descs_two);
+}
+
TEST_SUITE(nffs_suite_cache)
{
int rc;
@@ -2541,6 +2609,7 @@ nffs_test_gen(void)
nffs_test_lost_found();
nffs_test_readdir();
nffs_test_split_file();
+ nffs_test_gc_on_oom();
}
TEST_SUITE(gen_1_1)